From 574dba0a64efec7ff4957b18e8dbfee63b85fde1 Mon Sep 17 00:00:00 2001 From: x Date: Tue, 29 Sep 2026 12:21:14 +0200 Subject: [PATCH 01/26] CultNet publishes every committed cache change once CultNetDatabase watched OnUpdate, which fires for loads alone, so cache.Commit, cache.UpsertAsync, variant dependents re-resolved by a base edit and Flatten never reached subscribers or the mutation log, and a load removal carried an empty key. The database now observes cache.Watch, the one owner of admitted changes. A write door of the database (PutAsync, DeleteAsync, ApplyPut/Delete, replica apply, sync) marks the key it is committing and publishes that change itself, since it carries the shard-log entry; every other change publishes from the cache, keyed. Co-Authored-By: Claude Sonnet 5 --- .../Contracts/cultcache-store-composition.md | 10 +- src/GameCult.Networking/CultNetDatabase.cs | 115 +++++++++++------- .../NetworkingTests.cs | 5 +- 3 files changed, 80 insertions(+), 50 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index ab028fca..e617199c 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -47,10 +47,12 @@ which route wrote it. - Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a per-cache, in-memory number assigned when the cache admits the change, under its gate. It increases in admission order and is not persisted. - `OnUpdate(previous, current)` carries none. Streams CultNet derives from - `OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server, - database-backed Mesh handles) have no ordering guarantee and no stale - protection. + `OnUpdate(previous, current)` carries none. CultNet derives its + streams (`CultNetDatabase.WatchAllChanges`, the subscription server, + database-backed Mesh handles) from `Watch`, so a load, a commit and a variant + dependent's re-resolution all reach subscribers and the mutation log; a + database write door publishes its own change, at its own key, itself. The + streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at or below it is reflected in the document. diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index d59cd6c7..885f7286 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -483,6 +483,8 @@ namespace GameCult.Networking private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); + private readonly IDisposable _cacheChanges; + private static readonly AsyncLocal<(string SchemaId, string Key)?> DoorOwned = new(); private bool _disposed; /// @@ -502,7 +504,7 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - _cache.OnUpdate += PublishCacheUpdate; + _cacheChanges = _cache.Watch().Subscribe(PublishCacheChange); } /// @@ -700,7 +702,8 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - RemoveTrackedDocument(existing.DocumentType, existing.Key); + using (Door(existing.SchemaId, existing.Key)) + _cache.Remove(existing.Key); PublishUntyped( existing.DocumentType, CultNetDatabaseChangeKind.Removed, @@ -716,15 +719,17 @@ namespace GameCult.Networking var key = new CultRecordKey(document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(document); var previous = _cache.Get(key); - var applied = await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document, - ShardId = shard.ShardId, - ShardEpoch = shard.Epoch - }).ConfigureAwait(false); + object applied; + using (Door(descriptor.SchemaId, key)) + applied = await _documents.ApplyRawDocumentPutMessageAsync( + _cache, + new CultNetDocumentPutRawMessage + { + MessageId = response.MessageId, + Document = document, + ShardId = shard.ShardId, + ShardEpoch = shard.Epoch + }).ConfigureAwait(false); PublishUntyped( descriptor.DocumentType, previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, @@ -847,7 +852,9 @@ namespace GameCult.Networking EnsurePrimary(shard, descriptor.SchemaId, key); var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); + CultRecordHandle handle; + using (Door(descriptor.SchemaId, key)) + handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); void PublishCommittedPut() { AppendMutationLogEntry( @@ -882,7 +889,9 @@ namespace GameCult.Networking EnsureClientAuthority(descriptor, key); var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); + CultRecordHandle handle; + using (Door(descriptor.SchemaId, key)) + handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; Publish(new CultNetDatabaseChange( CultNetDatabaseChangeKind.Predicted, @@ -907,7 +916,8 @@ namespace GameCult.Networking var previous = _cache.Get(key); if (previous != null) { - _cache.Remove(new CultRecordHandle(key)); + using (Door(descriptor.SchemaId, key)) + _cache.Remove(new CultRecordHandle(key)); void PublishCommittedDelete() { AppendMutationLogEntry( @@ -948,7 +958,9 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); + object document; + using (Door(descriptor.SchemaId, key)) + document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; var predictionKey = PredictionKey(descriptor.SchemaId, key); var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); @@ -1004,7 +1016,8 @@ namespace GameCult.Networking return Task.CompletedTask; } - _cache.Remove(key); + using (Door(descriptor.SchemaId, key)) + _cache.Remove(key); void PublishCommittedDelete() { AppendMutationLogEntry( @@ -1154,13 +1167,17 @@ namespace GameCult.Networking } _disposed = true; - _cache.OnUpdate -= PublishCacheUpdate; + _cacheChanges.Dispose(); _changes.Dispose(); } - private void PublishCacheUpdate(object? previous, object? current) + // The cache is the one owner of "a change was committed". Every admitted change reaches subscribers and the mutation + // log through here, except the change a write door of this database is itself committing at its own key: that door + // publishes it with the shard-log entry it must carry (Reconciled, a replica's wire entry). Changes at other keys + // in the same hold, such as variant dependents re-resolved by a base edit, are not the door's and publish here. + private void PublishCacheChange(CultCacheDocumentChange change) { - var document = current ?? previous; + var document = change.Document ?? change.PreviousDocument; if (document == null) { return; @@ -1168,26 +1185,38 @@ namespace GameCult.Networking var documentType = document.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); - var key = current != null - ? GetTrackedKey(current, documentType) - : new CultRecordKey(string.Empty); - var shard = ResolveShardInternal(descriptor, key); - var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); - var change = Activator.CreateInstance( - changeType, - current == null ? CultNetDatabaseChangeKind.Removed : - previous == null ? CultNetDatabaseChangeKind.Added : - CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - current, - previous, - null); - if (change != null) + if (DoorOwned.Value is { } door && + door.SchemaId == descriptor.SchemaId && + door.Key == change.Key.Value) { - _changes.OnNext(change); + return; } + + var kind = change.Kind switch + { + CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, + CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, + _ => CultNetDatabaseChangeKind.Updated + }; + var shard = ResolveShardInternal(descriptor, change.Key); + AppendMutationLogEntry(shard, kind, change.Key, descriptor.SchemaId, change.Document, change.PreviousDocument); + PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); + } + + private static DoorScope Door(string schemaId, CultRecordKey key) + { + var scope = new DoorScope(DoorOwned.Value); + DoorOwned.Value = (schemaId, key.Value); + return scope; + } + + private readonly struct DoorScope : IDisposable + { + private readonly (string SchemaId, string Key)? _outer; + + public DoorScope((string SchemaId, string Key)? outer) => _outer = outer; + + public void Dispose() => DoorOwned.Value = _outer; } private CultRecordKey GetTrackedKey(object document, Type documentType) @@ -1240,11 +1269,6 @@ namespace GameCult.Networking } } - private void RemoveTrackedDocument(Type documentType, CultRecordKey key) - { - _cache.Remove(key); - } - private void Publish(CultNetDatabaseChange change) where T : class { _changes.OnNext(change); @@ -1454,7 +1478,9 @@ namespace GameCult.Networking var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); + object document; + using (Door(descriptor.SchemaId, key)) + document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { @@ -1513,7 +1539,8 @@ namespace GameCult.Networking var previous = _cache.Get(key); if (previous != null) { - _cache.Remove(key); + using (Door(descriptor.SchemaId, key)) + _cache.Remove(key); } RecordMutationLogEntry(new CultNetShardMutationLogEntry( diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 40bf43eb..ab54a260 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1525,7 +1525,7 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(batched), Is.Null); }), Is.True); - Assert.That(published, Is.EqualTo(new[] { put })); + Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it"); Assert.That(cache.Get(batched)?.Text, Is.EqualTo("batched")); var reopened = new CultCache(); reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); @@ -2254,7 +2254,8 @@ namespace GameCult.Networking.Tests var cache = new CultCache(); var delivering = new ManualResetEventSlim(); - cache.OnUpdate += (_, _) => delivering.Set(); + // Subscribed before the database, so it runs before the database's own observer of the same change. + using var loadDelivering = cache.Watch().Subscribe(_ => delivering.Set()); var database = new CultNetDatabase(cache); var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { From f3180f019454c679fc5761889d762618a7c05e0c Mon Sep 17 00:00:00 2001 From: x Date: Tue, 29 Sep 2026 12:21:14 +0200 Subject: [PATCH 02/26] Pin CultNet publication of cache commits over RUDP Commit and upsert, database put/delete, replicated put, load removal and a base edit re-resolving a variant dependent, each observed by a real subscriber. Co-Authored-By: Claude Sonnet 5 --- .../CultNetPublishesCommitsTests.cs | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs diff --git a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs new file mode 100644 index 00000000..08ef57cb --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs @@ -0,0 +1,230 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; +using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests; + +namespace GameCult.Networking.Tests +{ + // The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the + // mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every + // other change in the hold publishes from the cache. + public sealed class CultNetPublishesCommitsTests + { + private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) => + wire.Changes.Where(change => KeyOf(change) == key).ToArray(); + + private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey; + + private sealed class Wire : IDisposable + { + private readonly CancellationTokenSource _cancellation = new(); + private readonly Thread _thread; + private readonly RudpCultNetSchemaServer _server; + private readonly CultNetDatabaseSubscriptionServer _subscriptions; + private readonly ICultNetSchemaClient _client; + public ConcurrentQueue Changes { get; } = new(); + public int Snapshots; + public ConcurrentQueue Errors { get; } = new(); + + private readonly CultNetDatabase _database; + + public Wire(CultNetDatabase database) + { + _database = database; + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + _server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket }); + _subscriptions = new CultNetDatabaseSubscriptionServer(_server, database); + _thread = new Thread(() => + { + while (!_cancellation.IsCancellationRequested) + { + _ = _server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + _thread.Start(); + _client = CultNetSchemaClients.CreateRudp("publish-client"); + _client.OnCultNet(Changes.Enqueue); + _client.OnCultNet(Errors.Enqueue); + _client.OnCultNet(_ => Interlocked.Increment(ref Snapshots)); + _client.Connect("127.0.0.1", _server.LocalEndPoint.Port); + } + + public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes) + { + await Until(() => _client.Connected); + _client.SendCultNet(new CultNetDatabaseSubscribeMessage + { + MessageId = "subscribe", + SubscriptionId = "all", + SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(), + IncludeSnapshot = includeSnapshot + }); + await Until(() => Snapshots > 0 || !Errors.IsEmpty); + Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused"); + } + + public void Dispose() + { + _cancellation.Cancel(); + _thread.Join(); + _client.Dispose(); + _subscriptions.Dispose(); + _server.Dispose(); + } + } + + private static async Task Until(Func condition) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + Assert.Fail("Condition was not satisfied before timeout."); + await Task.Delay(5); + } + } + + // One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance + // to arrive twice. + private static async Task SettleAsync(CultNetDatabase database, Wire wire) + { + var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N"); + await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" }); + await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel)); + } + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + [Test] + public async Task ACacheCommitAndUpsertReachASubscriberOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:commit"); + + cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle(key))); + await Until(() => Of(wire, key.Value).Length >= 1); + cache.Commit(batch => batch.Remove(key)); + await Until(() => Of(wire, key.Value).Length >= 2); + await cache.UpsertAsync(Note("upserted"), new CultRecordHandle(key)); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3)); + } + + [Test] + public async Task ADatabasePutAndDeleteStillPublishExactlyOnce() + { + var database = new CultNetDatabase(new CultCache()); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:put"); + + await database.PutAsync(key, Note("put")); + await database.DeleteAsync(key); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + [Test] + public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:replicated"); + var message = database.Documents.CreateRawDocumentPutMessage( + "put-replicated", new CultRecordHandle(key), Note("remote")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.ApplyPutAsync(message); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1)); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1)); + } + + [Test] + public async Task ALoadRemovalCarriesItsKey() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc"); + try + { + var key = new CultRecordKey("tests:publish:loaded"); + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("there"), new CultRecordHandle(key)); + writer.FlushAllBackingStores(); + + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var database = new CultNetDatabase(cache); + Assert.That(cache.Get(key), Is.Not.Null); + using var wire = new Wire(database); + await wire.SubscribeAsync(true, typeof(NetworkSchemaNote)); + + writer.Remove(key); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big"), + cache.Override(nameof(VariantWireFixture.Power), 9) + }); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(VariantWireFixture)); + var errorsBefore = wire.Errors.Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + await Until(() => wire.Errors.Count > errorsBefore); + await Until(() => Of(wire, baseKey.Value).Length >= 1); + + Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value), + "the re-resolved dependent is published, then refused (Q6) by key"); + Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1)); + Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record"); + } + } +} From f3343c86b4d8d5ba24e36f9cd996433e6ae45f6d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:21:39 +0200 Subject: [PATCH 03/26] CultNet: the Watch handler is the only publisher and logger The previous cut left two owners of publication: doors published and logged their own change while a flow-scoped key mark told the Watch handler to skip it. A write an observer made to the door key inside that scope was dropped, a door change and its re-resolved variant were logged in the wrong order, and log state was unlocked. Doors now register the context they alone know (wire message, reconciled prediction, replicated entry) under the identity of the instance they write or remove; the Watch handler consumes it once for that change and is the sole publisher and logger. Only a shard primary mints log sequences; a replica logs only the primary entries its replica door hands over. Log state sits under one gate. The AsyncLocal mark and every door-side publish or append are deleted. Co-Authored-By: Claude Sonnet 5 --- .../Contracts/cultcache-store-composition.md | 6 +- src/GameCult.Networking/CultNetDatabase.cs | 385 +++++++----------- 2 files changed, 147 insertions(+), 244 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index e617199c..fb18ca73 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -50,8 +50,10 @@ which route wrote it. `OnUpdate(previous, current)` carries none. CultNet derives its streams (`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed Mesh handles) from `Watch`, so a load, a commit and a variant - dependent's re-resolution all reach subscribers and the mutation log; a - database write door publishes its own change, at its own key, itself. The + dependent's re-resolution all reach subscribers and the mutation log through + one handler over `Watch`; a database write door only hands that handler the + wire message or replicated entry for the change it admitted. A replica + publishes but never mints log sequences. The streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 885f7286..f79108c3 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using GameCult.Caching; @@ -484,7 +486,15 @@ namespace GameCult.Networking new(); private readonly Subject _changes = new(); private readonly IDisposable _cacheChanges; - private static readonly AsyncLocal<(string SchemaId, string Key)?> DoorOwned = new(); + // What a write door of this database knows about the change it is admitting and the cache does not: the wire + // message it carries, a prediction it reconciles, the replicated log entry it applies. The cache's Watch stream + // is the only publisher and the only logger; a door registers this context under the identity of the change it + // admits (the instance it writes, or the instance it removes) and the stream handler consumes it once, for that + // change alone. A write to the same key by anyone else is a different instance and gets no context. + private readonly ConcurrentDictionary _doors = new(ReferenceIdentity.Instance); + // One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a + // subscriber may write the cache from its handler. + private readonly object _logGate = new(); private bool _disposed; /// @@ -559,18 +569,21 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - if (!_mutationLogs.TryGetValue(shardId, out var entries)) + lock (_logGate) { - return Array.Empty(); - } + if (!_mutationLogs.TryGetValue(shardId, out var entries)) + { + return Array.Empty(); + } - var query = entries.Where(entry => entry.Sequence > afterSequence); - if (limit.HasValue) - { - query = query.Take(limit.Value); - } + var query = entries.Where(entry => entry.Sequence > afterSequence); + if (limit.HasValue) + { + query = query.Take(limit.Value); + } - return query.ToArray(); + return query.ToArray(); + } } /// @@ -702,42 +715,17 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - using (Door(existing.SchemaId, existing.Key)) + using (Door(existing.Document, new DoorContext(shard) { NoLog = true })) _cache.Remove(existing.Key); - PublishUntyped( - existing.DocumentType, - CultNetDatabaseChangeKind.Removed, - existing.Key, - existing.SchemaId, - shard, - document: null, - previousDocument: existing.Document); } foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document))) { var key = new CultRecordKey(document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(document); - var previous = _cache.Get(key); - object applied; - using (Door(descriptor.SchemaId, key)) - applied = await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document, - ShardId = shard.ShardId, - ShardEpoch = shard.Epoch - }).ConfigureAwait(false); - PublishUntyped( - descriptor.DocumentType, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - applied, - previous); + var applied = _documents.DeserializeRawDocument(document); + using (Door(applied, new DoorContext(shard) { NoLog = true })) + await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false); } var sequence = response.ShardLogSequence ?? 0; @@ -851,29 +839,7 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key); - var previous = _cache.Get(key); - CultRecordHandle handle; - using (Door(descriptor.SchemaId, key)) - handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - void PublishCommittedPut() - { - AppendMutationLogEntry( - shard, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - document, - previous); - Publish(new CultNetDatabaseChange( - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - document, - previous)); - } - PublishCommittedPut(); - return handle; + return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } /// @@ -888,18 +854,10 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsureClientAuthority(descriptor, key); - var previous = _cache.Get(key); CultRecordHandle handle; - using (Door(descriptor.SchemaId, key)) + using (Door(document, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Predicted, NoLog = true })) handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Predicted, - key, - descriptor.SchemaId, - shard, - document, - previous)); return handle; } @@ -913,31 +871,7 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key); - var previous = _cache.Get(key); - if (previous != null) - { - using (Door(descriptor.SchemaId, key)) - _cache.Remove(new CultRecordHandle(key)); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous); - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous)); - } - PublishCommittedDelete(); - } - + _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; } @@ -957,44 +891,17 @@ namespace GameCult.Networking var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); - var previous = _cache.Get(key); - object document; - using (Door(descriptor.SchemaId, key)) - document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); - var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; + var document = _documents.DeserializeRawDocument(message.Document); var predictionKey = PredictionKey(descriptor.SchemaId, key); var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); + using (Door(document, new DoorContext(shard) + { + Put = message, + Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null + })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); if (reconcilesPrediction) - { - kind = CultNetDatabaseChangeKind.Reconciled; - } - - void PublishCommittedPut() - { - if (reconcilesPrediction) - _predictedDocuments.Remove(predictionKey); - AppendMutationLogEntry( - shard, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - key, - descriptor.SchemaId, - document, - previous, - new CultNetShardLogEntryMessage - { - ChangeKind = previous == null ? "added" : "updated", - Put = message - }); - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); - } - PublishCommittedPut(); + _predictedDocuments.Remove(predictionKey); return document; } @@ -1016,32 +923,8 @@ namespace GameCult.Networking return Task.CompletedTask; } - using (Door(descriptor.SchemaId, key)) + using (Door(previous, new DoorContext(shard) { Delete = message })) _cache.Remove(key); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous, - wireEntry: new CultNetShardLogEntryMessage - { - ChangeKind = "removed", - Delete = message - }); - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); - } - PublishCommittedDelete(); return Task.CompletedTask; } @@ -1171,52 +1054,110 @@ namespace GameCult.Networking _changes.Dispose(); } - // The cache is the one owner of "a change was committed". Every admitted change reaches subscribers and the mutation - // log through here, except the change a write door of this database is itself committing at its own key: that door - // publishes it with the shard-log entry it must carry (Reconciled, a replica's wire entry). Changes at other keys - // in the same hold, such as variant dependents re-resolved by a base edit, are not the door's and publish here. + // The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted + // change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in the + // order the cache admitted it within a hold. A replica publishes what it admits but never mints log sequences: its + // log holds only the primary's own entries, which the replica door hands over. private void PublishCacheChange(CultCacheDocumentChange change) { - var document = change.Document ?? change.PreviousDocument; - if (document == null) + var identity = change.Document ?? change.PreviousDocument; + if (identity == null) { return; } - var documentType = document.GetType(); + _doors.TryRemove(identity, out var door); + var documentType = identity.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); - if (DoorOwned.Value is { } door && - door.SchemaId == descriptor.SchemaId && - door.Key == change.Key.Value) - { - return; - } - - var kind = change.Kind switch + var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); + var kind = door?.Kind ?? change.Kind switch { CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, _ => CultNetDatabaseChangeKind.Updated }; - var shard = ResolveShardInternal(descriptor, change.Key); - AppendMutationLogEntry(shard, kind, change.Key, descriptor.SchemaId, change.Document, change.PreviousDocument); + var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + if (door?.Replica is { } replicated) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + replicated.Sequence, + replicated.CommittedAt, + logKind, + descriptor.SchemaId, + change.Key, + change.Document, + change.PreviousDocument), + replicated); + _lastWriteSequence[(descriptor.SchemaId, change.Key.Value)] = replicated.Sequence; + } + } + else if (door?.NoLog != true && shard.IsPrimary) + { + AppendMutationLogEntry( + shard, + logKind, + change.Key, + descriptor.SchemaId, + change.Document, + change.PreviousDocument, + door?.Put != null + ? new CultNetShardLogEntryMessage + { + ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated", + Put = door.Put + } + : door?.Delete != null + ? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete } + : null); + } + PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } - private static DoorScope Door(string schemaId, CultRecordKey key) + private DoorRegistration Door(object identity, DoorContext context) { - var scope = new DoorScope(DoorOwned.Value); - DoorOwned.Value = (schemaId, key.Value); - return scope; + _doors[identity] = context; + return new DoorRegistration(_doors, identity); } - private readonly struct DoorScope : IDisposable + // Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind. + private readonly struct DoorRegistration : IDisposable { - private readonly (string SchemaId, string Key)? _outer; + private readonly ConcurrentDictionary _doors; + private readonly object _identity; - public DoorScope((string SchemaId, string Key)? outer) => _outer = outer; + public DoorRegistration(ConcurrentDictionary doors, object identity) + { + _doors = doors; + _identity = identity; + } - public void Dispose() => DoorOwned.Value = _outer; + public void Dispose() => _doors.TryRemove(_identity, out _); + } + + private sealed class DoorContext + { + public DoorContext(CultNetShardDescriptor shard) => Shard = shard; + + public CultNetShardDescriptor Shard { get; } + public CultNetDatabaseChangeKind? Kind { get; set; } + public bool NoLog { get; set; } + public CultNetDocumentPutRawMessage? Put { get; set; } + public CultNetDocumentDeleteMessage? Delete { get; set; } + public CultNetShardLogEntryMessage? Replica { get; set; } + } + + private sealed class ReferenceIdentity : IEqualityComparer + { + public static readonly ReferenceIdentity Instance = new(); + + bool IEqualityComparer.Equals(object? x, object? y) => ReferenceEquals(x, y); + + int IEqualityComparer.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value); } private CultRecordKey GetTrackedKey(object document, Type documentType) @@ -1269,11 +1210,6 @@ namespace GameCult.Networking } } - private void Publish(CultNetDatabaseChange change) where T : class - { - _changes.OnNext(change); - } - private sealed class TrackedShardDocument { public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document) @@ -1299,6 +1235,8 @@ namespace GameCult.Networking object? previousDocument, CultNetShardLogEntryMessage? wireEntry = null) { + lock (_logGate) + { var sequence = NextMutationLogSequence(shard.ShardId); var committedAt = DateTimeOffset.UtcNow.ToString("O"); var entry = new CultNetShardMutationLogEntry( @@ -1316,6 +1254,7 @@ namespace GameCult.Networking : wireEntry ?? ToLogEntryMessage(entry); RecordMutationLogEntry(entry, storedWireEntry); _lastWriteSequence[(schemaId, key.Value)] = sequence; + } } /// @@ -1323,8 +1262,11 @@ namespace GameCult.Networking /// never been committed through this database. The order the CultNet selection evaluator /// pages by (docs/cultnet-selection-cut.md, section 2). /// - public long? LastWriteSequence(string schemaId, CultRecordKey key) => - _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + public long? LastWriteSequence(string schemaId, CultRecordKey key) + { + lock (_logGate) + return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + } /// /// The current write-sequence watermark across every shard this database holds, 0 when nothing @@ -1335,7 +1277,11 @@ namespace GameCult.Networking /// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to /// while the page's own shard sat still. /// - public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + public ulong CurrentAsOf() + { + lock (_logGate) + return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + } /// /// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit @@ -1349,7 +1295,8 @@ namespace GameCult.Networking public ulong CurrentAsOf(string shardId) { if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; + lock (_logGate) + return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; } private void RecordMutationLogEntry( @@ -1477,10 +1424,6 @@ namespace GameCult.Networking var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); - var previous = _cache.Get(key); - object document; - using (Door(descriptor.SchemaId, key)) - document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { @@ -1494,30 +1437,11 @@ namespace GameCult.Networking kind = CultNetDatabaseChangeKind.Reconciled; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - descriptor.SchemaId, - key, - document, - previous), - entry); - // R-Q: a replica apply is a commit path too - the last-write sequence this row now carries - // is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets - // it, so the selection evaluator's order and cursor agree whether a row's most recent write - // landed locally or arrived through replication. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); + // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that + // admits the change; the stream handler does both. + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } private void ApplyCommittedDelete( @@ -1535,35 +1459,12 @@ namespace GameCult.Networking } var key = new CultRecordKey(message.RecordKey); - var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); var previous = _cache.Get(key); if (previous != null) { - using (Door(descriptor.SchemaId, key)) + using (Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry })) _cache.Remove(key); } - - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - CultNetDatabaseChangeKind.Removed, - descriptor.SchemaId, - key, - document: null, - previousDocument: previous), - entry); - // R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) From d8d973c4185bc7dfbc8a8c1743bce3f5e9d51cd6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:21:39 +0200 Subject: [PATCH 04/26] Pin the single-owner publication cases Soul falsified Observer rewrite of the door key, variant log order, replica log sequences with and without a local write, 4x5000 concurrent writers, a write after a database delete, shard-snapshot resync, and a disposed database. Co-Authored-By: Claude Sonnet 5 --- .../CultNetPublishesCommitsTests.cs | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs index 08ef57cb..fbd1df2a 100644 --- a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; using NUnit.Framework; +using R3; using static GameCult.Networking.Tests.NetworkingTests; using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests; @@ -226,5 +227,203 @@ namespace GameCult.Networking.Tests Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1)); Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record"); } + + // An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own. + [Test] + public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:normalized"); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "raw") + cache.UpsertAsync(Note("normalized"), new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + }); + + await database.PutAsync(key, Note("raw")); + await Until(() => Of(wire, key.Value).Length >= 2); + await SettleAsync(database, wire); + + Assert.That(cache.Get(key)!.Text, Is.EqualTo("normalized")); + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + return (cache, documents); + } + + private static System.Collections.Generic.List Record(CultNetDatabase database, Func describe) + { + var seen = new System.Collections.Generic.List(); + database.WatchAllChanges().Subscribe(change => seen.Add(describe(change))); + return seen; + } + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + [Test] + public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder() + { + var (cache, documents) = VariantCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big") + }); + var published = Record(database, KeyOfChange); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value), + Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + } + + private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) => + new(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + Shards = + [ + new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId]) + ] + }); + + [Test] + public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally() + { + var (sourceCache, sourceDocuments) = VariantCache(); + var (replicaCache, replicaDocuments) = VariantCache(); + var schemaId = sourceCache.Registry.GetRequired().SchemaId; + var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions + { + DocumentRegistry = sourceDocuments, + Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])] + }); + var replica = Replica(replicaCache, replicaDocuments, schemaId); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[] + { + replicaCache.Override(nameof(VariantWireFixture.Name), "big") + }); + await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Entries = source.GetMutationLogMessages("replica-shard").ToArray() + }); + + Assert.That(replicaCache.Get(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved"); + Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }), + "the replica log holds the primary's entry and mints none of its own"); + } + + [Test] + public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var published = Record(replica, KeyOfChange); + + await cache.UpsertAsync(Note("local"), new CultRecordHandle(new CultRecordKey("tests:publish:local"))); + + Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + const int writers = 4, perWriter = 5000; + + Parallel.For(0, writers, writer => + { + for (var i = 0; i < perWriter; i++) + cache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey($"tests:publish:c:{writer}:{i}"))) + .GetAwaiter().GetResult(); + }); + + var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray(); + Assert.That(sequences, Has.Length.EqualTo(writers * perWriter)); + Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter)); + } + + [Test] + public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("tests:publish:after-delete"); + await database.PutAsync(key, Note("one")); + await database.DeleteAsync(key); + var published = Record(database, KeyOfChange); + + await cache.UpsertAsync(Note("two"), new CultRecordHandle(key)); + + Assert.That(published, Is.EqualTo(new[] { key.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind), + Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added + })); + } + + [Test] + public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var stale = new CultRecordKey("tests:publish:stale"); + var fresh = new CultRecordKey("tests:publish:fresh"); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(stale)); + var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value); + var put = replica.Documents.CreateRawDocumentPutMessage( + "snapshot-fresh", new CultRecordHandle(fresh), Note("fresh")); + + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public async Task ADisposedDatabaseNoLongerObservesTheCache() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var published = Record(database, KeyOfChange); + database.Dispose(); + + await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("tests:publish:disposed"))); + + Assert.That(published, Is.Empty); + } } } From fb5ffc74834b042cde71a014ecbdc470d25bbcd4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:06:05 +0200 Subject: [PATCH 05/26] CultNet: release cache changes in Sequence order and match door contexts by change shape The handler now parks a change ahead of a gap and drains one thread at a time, so the log and stream follow cache admission order and a subscriber that writes the cache queues behind the change it handles. A door context matches only a change of its own shape: a put door wrote an instance no cache held, a removal door removes it, so neither a plain removal nor an in-place re-upsert can wear it. A replica records every primary entry even when the key is absent, so chained replicas see no gap. Predictions are locked and cleared by the handler before publication. Co-Authored-By: Claude Sonnet 5 --- .../Contracts/cultcache-store-composition.md | 5 +- src/GameCult.Networking/CultNetDatabase.cs | 202 ++++++++++++++++-- 2 files changed, 182 insertions(+), 25 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index fb18ca73..c40bb54e 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -52,8 +52,9 @@ which route wrote it. database-backed Mesh handles) from `Watch`, so a load, a commit and a variant dependent's re-resolution all reach subscribers and the mutation log through one handler over `Watch`; a database write door only hands that handler the - wire message or replicated entry for the change it admitted. A replica - publishes but never mints log sequences. The + wire message or replicated entry for the change it admitted. That handler releases + changes in cache `Sequence` order, so the log and the stream agree with the + cache. A replica publishes but never mints log sequences. The streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index f79108c3..359049f1 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -474,7 +474,7 @@ namespace GameCult.Networking private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; - private readonly Dictionary _predictedDocuments = new(StringComparer.Ordinal); + private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); @@ -495,6 +495,16 @@ namespace GameCult.Networking // One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a // subscriber may write the cache from its handler. private readonly object _logGate = new(); + // The cache admits a change to a key under its gate, but each thread reaches this handler after leaving it, in any + // order. Changes are therefore released strictly in cache Sequence order: a change ahead of a gap is parked until the + // gap fills, and one thread at a time drains the ready queue, outside every lock, so a subscriber that writes the + // cache from its handler queues behind the change it is handling instead of running inside it. + private readonly SortedDictionary _parked = new(); + private readonly Queue _ready = new(); + private long _nextSequence; + private bool _draining; + // Predictions this runtime made and has not seen reconciled. Every read and write takes _predictionGate. + private readonly object _predictionGate = new(); private bool _disposed; /// @@ -514,7 +524,12 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - _cacheChanges = _cache.Watch().Subscribe(PublishCacheChange); + lock (_logGate) + { + _cacheChanges = _cache.Watch().Subscribe(OnCacheChange); + // Changes admitted before this read are released as they arrive; the next one is the first this handler waits for. + _nextSequence = _cache.GetWithSequence(new CultRecordKey(string.Empty)).Sequence + 1; + } } /// @@ -855,9 +870,14 @@ namespace GameCult.Networking EnsureClientAuthority(descriptor, key); CultRecordHandle handle; - using (Door(document, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Predicted, NoLog = true })) + using (Door(document, new DoorContext(shard) + { + Kind = CultNetDatabaseChangeKind.Predicted, + NoLog = true, + PredictionKey = PredictionKey(descriptor.SchemaId, key), + Predicts = true + })) handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; return handle; } @@ -893,15 +913,14 @@ namespace GameCult.Networking EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); var document = _documents.DeserializeRawDocument(message.Document); var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); + var reconcilesPrediction = HasPrediction(predictionKey); using (Door(document, new DoorContext(shard) { Put = message, - Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null + Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null, + PredictionKey = reconcilesPrediction ? predictionKey : null })) await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); - if (reconcilesPrediction) - _predictedDocuments.Remove(predictionKey); return document; } @@ -923,7 +942,7 @@ namespace GameCult.Networking return Task.CompletedTask; } - using (Door(previous, new DoorContext(shard) { Delete = message })) + using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true })) _cache.Remove(key); return Task.CompletedTask; } @@ -1055,18 +1074,76 @@ namespace GameCult.Networking } // The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted - // change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in the - // order the cache admitted it within a hold. A replica publishes what it admits but never mints log sequences: its - // log holds only the primary's own entries, which the replica door hands over. - private void PublishCacheChange(CultCacheDocumentChange change) + // change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in cache + // Sequence order. A replica publishes what it admits but never mints log sequences: its log holds only the primary's + // own entries, which the replica door hands over. + private void OnCacheChange(CultCacheDocumentChange change) { + var pending = new PendingChange(change, TakeDoor(change)); + lock (_logGate) + { + if (_disposed) + { + return; + } + + if (change.Sequence < _nextSequence) + { + _ready.Enqueue(pending); + } + else + { + _parked[change.Sequence] = pending; + while (_parked.Remove(_nextSequence, out var next)) + { + _ready.Enqueue(next); + _nextSequence++; + } + } + + if (_draining) + { + return; + } + + _draining = true; + } + + try + { + while (true) + { + PendingChange? item; + lock (_logGate) + { + if (_disposed || !_ready.TryDequeue(out item)) + { + _draining = false; + return; + } + } + + Deliver(item); + } + } + catch + { + lock (_logGate) + _draining = false; + throw; + } + } + + private void Deliver(PendingChange item) + { + var change = item.Change; + var door = item.Door; var identity = change.Document ?? change.PreviousDocument; if (identity == null) { return; } - _doors.TryRemove(identity, out var door); var documentType = identity.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); @@ -1077,6 +1154,17 @@ namespace GameCult.Networking _ => CultNetDatabaseChangeKind.Updated }; var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + if (door?.PredictionKey is { } predictionKey) + { + lock (_predictionGate) + { + if (door.Predicts) + _predictedDocuments.Add(predictionKey); + else + _predictedDocuments.Remove(predictionKey); + } + } + if (door?.Replica is { } replicated) { lock (_logGate) @@ -1118,25 +1206,72 @@ namespace GameCult.Networking PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } - private DoorRegistration Door(object identity, DoorContext context) + private bool HasPrediction(string predictionKey) { - _doors[identity] = context; - return new DoorRegistration(_doors, identity); + lock (_predictionGate) + return _predictedDocuments.Contains(predictionKey); } + // A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had + // held, so its change carries that instance as the new document and never as the previous one; a removal door + // removes an instance, so its change carries it as the previous document and no new one. A plain removal of an + // instance a put door admitted, or an observer re-upserting that instance in place, is neither, and gets no context. + private DoorContext? TakeDoor(CultCacheDocumentChange change) + { + var identity = change.Document ?? change.PreviousDocument; + if (identity == null || !_doors.TryGetValue(identity, out var door)) + { + return null; + } + + var matches = change.Document == null + ? door.Removal + : !door.Removal && !ReferenceEquals(change.PreviousDocument, change.Document); + return matches && ((ICollection>)_doors) + .Remove(new KeyValuePair(identity, door)) + ? door + : null; + } + + private DoorRegistration Door(object identity, DoorContext context) => + new(_doors, identity, context, _doors.TryAdd(identity, context)); + // Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind. private readonly struct DoorRegistration : IDisposable { private readonly ConcurrentDictionary _doors; private readonly object _identity; + private readonly DoorContext _context; + private readonly bool _owned; - public DoorRegistration(ConcurrentDictionary doors, object identity) + public DoorRegistration(ConcurrentDictionary doors, object identity, DoorContext context, bool owned) { _doors = doors; _identity = identity; + _context = context; + _owned = owned; } - public void Dispose() => _doors.TryRemove(_identity, out _); + // True when the stream handler took the context: the door's write was admitted. + public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context); + + public void Dispose() + { + if (_owned) + ((ICollection>)_doors).Remove(new KeyValuePair(_identity, _context)); + } + } + + private sealed class PendingChange + { + public PendingChange(CultCacheDocumentChange change, DoorContext? door) + { + Change = change; + Door = door; + } + + public CultCacheDocumentChange Change { get; } + public DoorContext? Door { get; } } private sealed class DoorContext @@ -1146,9 +1281,13 @@ namespace GameCult.Networking public CultNetShardDescriptor Shard { get; } public CultNetDatabaseChangeKind? Kind { get; set; } public bool NoLog { get; set; } + public bool Removal { get; set; } public CultNetDocumentPutRawMessage? Put { get; set; } public CultNetDocumentDeleteMessage? Delete { get; set; } public CultNetShardLogEntryMessage? Replica { get; set; } + // The prediction this door's change either records (Predicts) or ends. + public string? PredictionKey { get; set; } + public bool Predicts { get; set; } } private sealed class ReferenceIdentity : IEqualityComparer @@ -1432,7 +1571,8 @@ namespace GameCult.Networking } var predictionKey = PredictionKey(descriptor.SchemaId, key); - if (_predictedDocuments.Remove(predictionKey)) + var reconcilesPrediction = HasPrediction(predictionKey); + if (reconcilesPrediction) { kind = CultNetDatabaseChangeKind.Reconciled; } @@ -1440,7 +1580,7 @@ namespace GameCult.Networking // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that // admits the change; the stream handler does both. var document = _documents.DeserializeRawDocument(message.Document); - using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry })) + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = reconcilesPrediction ? predictionKey : null })) await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } @@ -1459,11 +1599,27 @@ namespace GameCult.Networking } var key = new CultRecordKey(message.RecordKey); + var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); + var admitted = false; var previous = _cache.Get(key); if (previous != null) { - using (Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry })) - _cache.Remove(key); + using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true }); + _cache.Remove(key); + admitted = door.Consumed; + } + + if (!admitted) + { + // The key is not here, so nothing was admitted to log. This replica's log must still hold every entry of the + // primary's, or a replica chained on it sees a gap. + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed, + descriptor.SchemaId, key, document: null, previousDocument: null), entry); + _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; + } } } From 28248ebedee098188d11a519ee2b55296f6a14e1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:06:05 +0200 Subject: [PATCH 06/26] Adopt Soul pass 2 probes and pin log-before-publish and in-place re-upsert Co-Authored-By: Claude Sonnet 5 --- .../CultNetPublicationOrderTests.cs | 347 ++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs diff --git a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs new file mode 100644 index 00000000..65e290a4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs @@ -0,0 +1,347 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + public sealed class CultNetPublicationOrderTests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text; + + // A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed + // (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile. + [Test] + public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "A") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache); + var seen = new List(); + using var subscriber = database.Watch().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); }); + var key = new CultRecordKey("soul:order"); + + var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle(key)).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True); + await database.PutAsync(key, Note("B")); + release.Set(); + await a; + + var shard = database.Shards[0].ShardId; + var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last(); + + // A replica built from the log. + var replicaCache = new CultCache(); + var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)] + }); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shard, + ShardEpoch = database.Shards[0].Epoch, + Entries = database.GetMutationLogMessages(shard).ToArray() + }); + + Assert.Multiple(() => + { + Assert.That(cache.Get(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B"); + Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write"); + Assert.That(replicaCache.Get(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary"); + Assert.That(seen.Last(), Is.EqualTo("B"), "a subscriber's final state for the key"); + }); + } + + // Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache. + [Test] + public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing() + { + var divergent = 0; + const int rounds = 300; + for (var round = 0; round < rounds; round++) + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:hammer"); + using var start = new Barrier(2); + Parallel.For(0, 2, writer => + { + start.SignalAndWait(); + for (var i = 0; i < 40; i++) + cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle(key)).GetAwaiter().GetResult(); + }); + var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last(); + if (!ReferenceEquals(last.Document, cache.Get(key))) + divergent++; + } + + TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}"); + Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache"); + } + + // A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in + // the window before the door's own change is handled, consumes the door's context. + [Test] + public async Task S2_ARemovalNeverWearsAReconcilingPutsContext() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:reconcile"); + await database.PutPredictedAsync(key, Note("predicted")); + var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>(); + using var subscriber = database.Watch().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text))); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + var a = Task.Run(() => database.ApplyPutAsync(message)); + Assert.That(entered.Wait(Wait), Is.True); + cache.Remove(key); + release.Set(); + await a; + + var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray(); + Assert.Multiple(() => + { + Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1), + "the removal is published as a removal: " + string.Join(", ", published)); + Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text), + Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document"); + Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True, + "no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document)))); + }); + } + + // A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not + // hold (B removed it locally, a supported write) must still keep B's log contiguous for C. + [Test] + public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey() + { + const string shardId = "chain"; + var schemaId = new CultCache().Registry.GetRequired().SchemaId; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])] + }); + var p = Make(new CultCache(), true); + var bCache = new CultCache(); + var b = Make(bCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray() + }); + } + + var k = new CultRecordKey("soul:chain:k"); + await p.PutAsync(k, Note("x")); + await Pull(p, b); + await Pull(b, c); + bCache.Remove(k); + await p.DeleteAsync(k); + await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y")); + await Pull(p, b); + + Assert.Multiple(() => + { + Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }), + "B's log holds every primary entry it applied"); + Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap"); + }); + } + + // A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction. + [Test] + public async Task S4_APredictionMadeInReactionToAReconcileSurvives() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:repredict"); + var kinds = new ConcurrentQueue(); + var repredicted = false; + using var subscriber = database.Watch().Subscribe(change => + { + kinds.Enqueue(change.Kind); + if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted) + { + repredicted = true; + database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult(); + } + }); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled + })); + } + // Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A + // refused prediction leaves nothing behind that a later write of the same instance would wear. + [Test] + public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var baseKey = new CultRecordKey("soul-refuse-base"); + var variantKey = new CultRecordKey("soul-refuse-variant"); + await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 }, + new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v") + }); + var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 }; + Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException); + var kinds = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => kinds.Enqueue(change.Kind)); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await cache.UpsertAsync(instance, new CultRecordHandle(new CultRecordKey("soul-refuse-other"))); + + Assert.Multiple(() => + { + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1)); + }); + } + + // Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a + // durable log store can observe even though the disposed Subject hides publication. + [Test] + public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog() + { + var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N")); + var store = new CultNetFileShardMutationLogStore(root); + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store }); + var shard = database.Shards[0].ShardId; + await cache.UpsertAsync(Note("before"), new CultRecordHandle(new CultRecordKey("soul:dispose:before"))); + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + database.Dispose(); + + try { await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("soul:dispose:after"))); } + catch { /* a throwing handler would also be a failure below */ } + + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + System.IO.Directory.Delete(root, true); + } + + // The log entry exists before its change is published: a subscriber that reads the log from its handler finds it. + [Test] + public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:log-first"); + var loggedWhenPublished = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => + loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value))); + + await database.PutAsync(key, Note("one")); + cache.Remove(key); + + Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true })); + } + + // An observer that mutates the admitted instance in place and upserts it again writes a second change of the same + // instance. It is published as its own update and never wears the context of the door's change, even when it + // reaches the database first. + [Test] + public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext() + { + var cache = new CultCache(); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + change.Document.Text = "normalized"; + cache.UpsertAsync(change.Document, new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:in-place"); + var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>(); + using var subscriber = database.Watch().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text))); + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(message); + + Assert.That(seen, Is.EqualTo(new (CultNetDatabaseChangeKind, string?)[] + { + (CultNetDatabaseChangeKind.Predicted, null), + (CultNetDatabaseChangeKind.Reconciled, "predicted"), + (CultNetDatabaseChangeKind.Updated, "normalized") + })); + } + } +} From fe7bca0cb5b30ba1fff43077d694b98c3dceb429 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:48:50 +0200 Subject: [PATCH 07/26] CultNetDatabase: log on the cache journal and publish from Watch, no queue Cut 4b-4d of the publication map. The reorder buffer, drainer, prediction gate and the logging inside the Watch handler are gone. The database registers the Watch observer, then one cache journal. The journal runs under the gate in Sequence order: it takes the door, decides the change kind and prediction bookkeeping, logs (a primary mints, a replica records the primary's entry) and stashes the change; the Watch observer publishes the stash entry by Sequence. A primary that cannot log a committed change burns that sequence, raises its own compaction floor, tries CompactThrough, and throws CultNetShardLogException (not an InvalidOperationException) after publication. A replica's refused append reaches the pull loop and the cursor does not advance. Fixes four defects the map named: the log is appended to disk before memory (and memory is a SortedList keyed by sequence); the snapshot reads its log sequence before its documents; a restart resumes above the compaction point; GetCompactedMutationLogSequence and GetLatestMutationLogSequence honour the in-memory floor. The snapshot removal door sets Removal. Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-store-composition.md | 25 +- src/GameCult.Networking/CultNetDatabase.cs | 491 ++++++++++-------- .../CultNetShardMutationLogStorage.cs | 5 + .../CultNetJournalTests.cs | 456 ++++++++++++++++ 4 files changed, 751 insertions(+), 226 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetJournalTests.cs diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 931088eb..a19a8839 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -47,14 +47,19 @@ which route wrote it. - Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a per-cache, in-memory number assigned when the cache admits the change, under its gate. It increases in admission order and is not persisted. - `OnUpdate(previous, current)` carries none. CultNet derives its - streams (`CultNetDatabase.WatchAllChanges`, the subscription server, - database-backed Mesh handles) from `Watch`, so a load, a commit and a variant - dependent's re-resolution all reach subscribers and the mutation log through - one handler over `Watch`; a database write door only hands that handler the - wire message or replicated entry for the change it admitted. That handler releases - changes in cache `Sequence` order, so the log and the stream agree with the - cache. A replica publishes but never mints log sequences. The + `OnUpdate(previous, current)` carries none. CultNet derives its streams + (`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed + Mesh handles) from `Watch` plus one journal (`AddJournal`), so a load, a + commit and a variant dependent's re-resolution all reach subscribers and the + mutation log. The journal runs under the gate in `Sequence` order, so the + shard log is written in cache order; the `Watch` observer publishes what the + journal decided. A database write door only hands the journal the wire message + or replicated entry for the change it admitted. A replica publishes but never + mints log sequences. A primary logs every change it admits, loads and pulls + included; a store attached after the database exists has its hydration logged + too. A primary that cannot log a change compacts through that change's + sequence and throws `CultNetShardLogException` to the writer after publication: + the commit stands, and a replica behind it resynchronizes from a snapshot. The streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at @@ -131,6 +136,10 @@ which route wrote it. needs the cache or its gate: another thread's write, a lock a writer holds. Its own leaf locks, taken only inside the journal, are the only locks that may nest under the gate. +- A shard mutation log store (`ICultNetShardMutationLogStore`) runs inside the + CultNet journal, under the cache's gate: it must not call the cache or wait + for a thread that does. A store that writes its own cache throws through the + journal's reentry guard, and the admission stands. - A thread holds one cache's gate at a time: entering a cache's hold inside another cache's hold throws. - A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 359049f1..202f88b5 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -463,6 +463,36 @@ namespace GameCult.Networking public string Reason { get; } } + /// + /// Raised to the writer when a shard primary committed a change and could not log it. The commit stands and the + /// change was published; the shard's log burned and compacted past it, so every replica + /// behind it resynchronizes from a snapshot. It is deliberately not an : + /// callers that read that type as a refused write must not mistake a committed one for it. When several things + /// fail in one admission, the writer sees an that contains this one. + /// + public sealed class CultNetShardLogException : Exception + { + /// + /// Creates a shard log exception. + /// + public CultNetShardLogException(string shardId, long sequence, Exception cause) + : base($"Shard '{shardId}' committed a change but could not log it; sequence {sequence} is burned and compacted past.", cause) + { + ShardId = shardId; + Sequence = sequence; + } + + /// + /// Gets the shard whose log refused the change. + /// + public string ShardId { get; } + + /// + /// Gets the shard-log sequence that was minted for the change and burned. + /// + public long Sequence { get; } + } + /// /// Database-style CultNet facade over a CultCache shard set. /// @@ -474,38 +504,39 @@ namespace GameCult.Networking private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; - private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); - private readonly Dictionary> _mutationLogs = + // Keyed by sequence, like the durable store: recording an entry twice replaces it. + private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); + // The highest sequence this primary burned per shard (a change it committed and could not log). Serves as the + // shard's compaction floor beside the store's own, so history the log cannot serve is answered compacted_history. + private readonly Dictionary _burnedThrough = new(StringComparer.Ordinal); private readonly Dictionary _appliedShardSequences = new(StringComparer.Ordinal); // CultNet typed selection, section 2: "ordinal is the sequence of the commit that last wrote - // the row". Kept on every append (AppendMutationLogEntry is the one site), keyed by (schemaId, + // the row". Kept on every append (LogPrimaryChange and RecordReplicatedEntry are the sites), keyed by (schemaId, // recordKey) so the evaluator's order and cursor never read a clock. private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); private readonly IDisposable _cacheChanges; + private readonly IDisposable _cacheJournal; // What a write door of this database knows about the change it is admitting and the cache does not: the wire - // message it carries, a prediction it reconciles, the replicated log entry it applies. The cache's Watch stream - // is the only publisher and the only logger; a door registers this context under the identity of the change it - // admits (the instance it writes, or the instance it removes) and the stream handler consumes it once, for that - // change alone. A write to the same key by anyone else is a different instance and gets no context. + // message it carries, a prediction it reconciles, the replicated log entry it applies. The journal is the only + // logger and consumes the context under the cache gate; a door registers it under the identity of the change it + // admits (the instance it writes, or the instance it removes), and the journal takes it once, for that change + // alone. A write to the same key by anyone else is a different instance and gets no context. private readonly ConcurrentDictionary _doors = new(ReferenceIdentity.Instance); - // One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a - // subscriber may write the cache from its handler. + // One gate over the mutation log, its sequence counters, the last-write map and the publication stash. Lock order: + // the cache gate, then this, then the log store's own lock. Nothing holding this calls the cache, and a log store + // must not either (ICultNetShardMutationLogStore). Publication stays outside it: a subscriber may write the cache + // from its handler. private readonly object _logGate = new(); - // The cache admits a change to a key under its gate, but each thread reaches this handler after leaving it, in any - // order. Changes are therefore released strictly in cache Sequence order: a change ahead of a gap is parked until the - // gap fills, and one thread at a time drains the ready queue, outside every lock, so a subscriber that writes the - // cache from its handler queues behind the change it is handling instead of running inside it. - private readonly SortedDictionary _parked = new(); - private readonly Queue _ready = new(); - private long _nextSequence; - private bool _draining; - // Predictions this runtime made and has not seen reconciled. Every read and write takes _predictionGate. - private readonly object _predictionGate = new(); - private bool _disposed; + // Changes the journal has decided and the Watch observer has not yet published, by cache Sequence. The journal + // runs under the cache gate before any observer, so each change is stashed before it can be published. + private readonly Dictionary _stash = new(); + // Predictions this runtime made and has not seen reconciled. Only the journal reads or writes it, under the cache gate. + private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); + private volatile bool _disposed; /// /// Creates a database surface over a CultCache instance. @@ -524,12 +555,9 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - lock (_logGate) - { - _cacheChanges = _cache.Watch().Subscribe(OnCacheChange); - // Changes admitted before this read are released as they arrive; the next one is the first this handler waits for. - _nextSequence = _cache.GetWithSequence(new CultRecordKey(string.Empty)).Sequence + 1; - } + // The observer first, the journal second: a change the journal stashed always finds its publisher. + _cacheChanges = _cache.Watch().Subscribe(Publish); + _cacheJournal = _cache.AddJournal(Journal); } /// @@ -591,7 +619,7 @@ namespace GameCult.Networking return Array.Empty(); } - var query = entries.Where(entry => entry.Sequence > afterSequence); + var query = entries.Values.Where(entry => entry.Sequence > afterSequence); if (limit.HasValue) { query = query.Take(limit.Value); @@ -628,7 +656,15 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _mutationLogStore?.GetCompactedThrough(shardId) ?? 0; + lock (_logGate) + return CompactedFloor(shardId); + } + + // The store's compaction point or this primary's own burned sequences, whichever is higher. Under _logGate. + private long CompactedFloor(string shardId) + { + var burned = _burnedThrough.TryGetValue(shardId, out var through) ? through : 0; + return Math.Max(burned, _mutationLogStore?.GetCompactedThrough(shardId) ?? 0); } /// @@ -638,11 +674,13 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - var compactedThrough = GetCompactedMutationLogSequence(shardId); - var retained = _mutationLogStore != null - ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) - : GetMutationLog(shardId).Select(entry => entry.Sequence); - return retained.DefaultIfEmpty(compactedThrough).Max(); + lock (_logGate) + { + var retained = _mutationLogStore != null + ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) + : GetMutationLog(shardId).Select(entry => entry.Sequence); + return retained.Append(CompactedFloor(shardId)).Max(); + } } /// @@ -659,6 +697,10 @@ namespace GameCult.Networking ThrowIfDisposed(); if (shard == null) throw new ArgumentNullException(nameof(shard)); + // The log sequence is read before any document: a write that lands in between is then replayed from the log on + // top of documents that already hold it (whole-document puts and deletes, applied in order), and a snapshot can + // never claim a sequence whose write its documents lack. + var logSequence = GetLatestMutationLogSequence(shard.ShardId); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); var lowKeys = CultNetV0SelectionLowering.Lower(filter?.RecordKeys); @@ -672,7 +714,7 @@ namespace GameCult.Networking Documents = Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -695,7 +737,7 @@ namespace GameCult.Networking Documents = page.Documents ?? Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -730,7 +772,7 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - using (Door(existing.Document, new DoorContext(shard) { NoLog = true })) + using (Door(existing.Document, new DoorContext(shard) { NoLog = true, Removal = true })) _cache.Remove(existing.Key); } @@ -912,14 +954,7 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); var document = _documents.DeserializeRawDocument(message.Document); - var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = HasPrediction(predictionKey); - using (Door(document, new DoorContext(shard) - { - Put = message, - Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null, - PredictionKey = reconcilesPrediction ? predictionKey : null - })) + using (Door(document, new DoorContext(shard) { Put = message, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); return document; } @@ -1057,7 +1092,6 @@ namespace GameCult.Networking (change.Document != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.Document)) || (change.PreviousDocument != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.PreviousDocument))); } - /// /// Releases database subscriptions. /// @@ -1070,146 +1104,182 @@ namespace GameCult.Networking _disposed = true; _cacheChanges.Dispose(); + _cacheJournal.Dispose(); + lock (_logGate) + _stash.Clear(); _changes.Dispose(); } - // The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted - // change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in cache - // Sequence order. A replica publishes what it admits but never mints log sequences: its log holds only the primary's - // own entries, which the replica door hands over. - private void OnCacheChange(CultCacheDocumentChange change) + // The cache is the one owner of "a change was committed"; the journal is this database's one decision point and its + // one logger. It runs under the cache gate, in cache Sequence order, before any observer: it takes the change's door, + // settles the change kind and the prediction bookkeeping, logs the change when this runtime is the shard's primary + // (a replica records only the primary's own entry, which its door carries), and stashes what the publisher needs. + // A change with no door was written by someone else (a bare cache write, a load, a pull) and is logged like any + // unattributed commit. It never enters the cache and calls nothing that does. + private void Journal(IReadOnlyList> changes) { - var pending = new PendingChange(change, TakeDoor(change)); - lock (_logGate) - { - if (_disposed) - { - return; - } - - if (change.Sequence < _nextSequence) - { - _ready.Enqueue(pending); - } - else - { - _parked[change.Sequence] = pending; - while (_parked.Remove(_nextSequence, out var next)) - { - _ready.Enqueue(next); - _nextSequence++; - } - } - - if (_draining) - { - return; - } - - _draining = true; - } - - try - { - while (true) - { - PendingChange? item; - lock (_logGate) - { - if (_disposed || !_ready.TryDequeue(out item)) - { - _draining = false; - return; - } - } - - Deliver(item); - } - } - catch - { - lock (_logGate) - _draining = false; - throw; - } - } - - private void Deliver(PendingChange item) - { - var change = item.Change; - var door = item.Door; - var identity = change.Document ?? change.PreviousDocument; - if (identity == null) + if (_disposed) { return; } - var documentType = identity.GetType(); - var descriptor = _cache.Registry.GetRequired(documentType); - var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); - var kind = door?.Kind ?? change.Kind switch + List? failures = null; + foreach (var change in changes) { - CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, - CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, - _ => CultNetDatabaseChangeKind.Updated - }; - var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; - if (door?.PredictionKey is { } predictionKey) - { - lock (_predictionGate) + var identity = change.Document ?? change.PreviousDocument; + if (identity == null) + { + continue; + } + + var door = TakeDoor(change); + var documentType = identity.GetType(); + var descriptor = _cache.Registry.GetRequired(documentType); + var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); + var kind = door?.Kind ?? change.Kind switch + { + CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, + CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, + _ => CultNetDatabaseChangeKind.Updated + }; + if (door?.PredictionKey is { } predictionKey) { if (door.Predicts) _predictedDocuments.Add(predictionKey); - else - _predictedDocuments.Remove(predictionKey); + else if (_predictedDocuments.Remove(predictionKey)) + kind = CultNetDatabaseChangeKind.Reconciled; } - } - if (door?.Replica is { } replicated) - { - lock (_logGate) + // The log records a reconciled change as the update it is. + var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + try { - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - replicated.Sequence, - replicated.CommittedAt, - logKind, - descriptor.SchemaId, - change.Key, - change.Document, - change.PreviousDocument), - replicated); - _lastWriteSequence[(descriptor.SchemaId, change.Key.Value)] = replicated.Sequence; + if (door?.Replica is { } replicated) + { + RecordReplicatedEntry(shard, logKind, descriptor.SchemaId, change, replicated); + } + else if (door?.NoLog != true && shard.IsPrimary) + { + LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door); + } } - } - else if (door?.NoLog != true && shard.IsPrimary) - { - AppendMutationLogEntry( - shard, - logKind, - change.Key, - descriptor.SchemaId, - change.Document, - change.PreviousDocument, - door?.Put != null - ? new CultNetShardLogEntryMessage - { - ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated", - Put = door.Put - } - : door?.Delete != null - ? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete } - : null); + catch (Exception exception) + { + (failures ??= new List()).Add(exception); + } + + lock (_logGate) + _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } - PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); + // Every change is stashed and will be published; the writer hears of a failed log afterwards. + if (failures != null) + { + throw failures.Count == 1 ? failures[0] : new AggregateException(failures); + } } - private bool HasPrediction(string predictionKey) + // The Watch observer publishes, on the writer's thread after the cache gate opens and before the write returns, what + // the journal stashed for this change. A change with no stash was admitted before this database existed. + private void Publish(CultCacheDocumentChange change) { - lock (_predictionGate) - return _predictedDocuments.Contains(predictionKey); + Publication? publication; + lock (_logGate) + { + if (!_stash.Remove(change.Sequence, out publication)) + { + return; + } + } + + PublishUntyped( + publication.DocumentType, + publication.Kind, + publication.Key, + publication.SchemaId, + publication.Shard, + publication.Document, + publication.PreviousDocument); + } + + private void RecordReplicatedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + CultNetShardLogEntryMessage replicated) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + replicated.Sequence, + replicated.CommittedAt, + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument), + replicated); + _lastWriteSequence[(schemaId, change.Key.Value)] = replicated.Sequence; + } + } + + // A primary mints the sequence, appends durably, then records in memory. If it cannot log the change, the change is + // still committed: the sequence is burned and compacted past, so a replica behind it is told compacted_history and + // resynchronizes by snapshot, and the writer is told with a CultNetShardLogException. + private void LogPrimaryChange( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + DoorContext? door) + { + lock (_logGate) + { + var sequence = NextMutationLogSequence(shard.ShardId); + _lastWriteSequence[(schemaId, change.Key.Value)] = sequence; + try + { + var entry = new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + sequence, + DateTimeOffset.UtcNow.ToString("O"), + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument); + var wireEntry = _mutationLogStore == null + ? null + : door?.Put != null + ? new CultNetShardLogEntryMessage + { + ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated", + Put = door.Put + } + : door?.Delete != null + ? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete } + : ToLogEntryMessage(entry); + RecordMutationLogEntry(entry, wireEntry); + } + catch (Exception cause) + { + _burnedThrough[shard.ShardId] = Math.Max(_burnedThrough.TryGetValue(shard.ShardId, out var burned) ? burned : 0, sequence); + try + { + _mutationLogStore?.CompactThrough(shard.ShardId, sequence); + } + catch (Exception compactCause) + { + cause = new AggregateException(cause, compactCause); + } + + throw new CultNetShardLogException(shard.ShardId, sequence, cause); + } + } } // A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had @@ -1252,7 +1322,7 @@ namespace GameCult.Networking _owned = owned; } - // True when the stream handler took the context: the door's write was admitted. + // True when the journal took the context: the door's write was admitted. public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context); public void Dispose() @@ -1262,16 +1332,34 @@ namespace GameCult.Networking } } - private sealed class PendingChange + // What the publisher needs of a change the journal decided. + private sealed class Publication { - public PendingChange(CultCacheDocumentChange change, DoorContext? door) + public Publication( + Type documentType, + CultNetDatabaseChangeKind kind, + CultRecordKey key, + string schemaId, + CultNetShardDescriptor shard, + object? document, + object? previousDocument) { - Change = change; - Door = door; + DocumentType = documentType; + Kind = kind; + Key = key; + SchemaId = schemaId; + Shard = shard; + Document = document; + PreviousDocument = previousDocument; } - public CultCacheDocumentChange Change { get; } - public DoorContext? Door { get; } + public Type DocumentType { get; } + public CultNetDatabaseChangeKind Kind { get; } + public CultRecordKey Key { get; } + public string SchemaId { get; } + public CultNetShardDescriptor Shard { get; } + public object? Document { get; } + public object? PreviousDocument { get; } } private sealed class DoorContext @@ -1365,36 +1453,6 @@ namespace GameCult.Networking public object Document { get; } } - private void AppendMutationLogEntry( - CultNetShardDescriptor shard, - CultNetDatabaseChangeKind kind, - CultRecordKey key, - string schemaId, - object? document, - object? previousDocument, - CultNetShardLogEntryMessage? wireEntry = null) - { - lock (_logGate) - { - var sequence = NextMutationLogSequence(shard.ShardId); - var committedAt = DateTimeOffset.UtcNow.ToString("O"); - var entry = new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - sequence, - committedAt, - kind, - schemaId, - key, - document, - previousDocument); - var storedWireEntry = _mutationLogStore == null - ? null - : wireEntry ?? ToLogEntryMessage(entry); - RecordMutationLogEntry(entry, storedWireEntry); - _lastWriteSequence[(schemaId, key.Value)] = sequence; - } - } /// /// The shard-log sequence of the commit that last wrote this row, or null when the row has @@ -1444,21 +1502,22 @@ namespace GameCult.Networking { if (!_mutationLogs.TryGetValue(entry.ShardId, out var entries)) { - entries = new List(); + entries = new SortedList(); _mutationLogs[entry.ShardId] = entries; } - entries.Add(entry); + // Disk first: memory never serves an entry the durable log lacks, and a refused append records nothing. + if (_mutationLogStore != null && wireEntry != null) + { + _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); + } + + entries[entry.Sequence] = entry; if (!_nextLogSequences.TryGetValue(entry.ShardId, out var next) || next <= entry.Sequence) { _nextLogSequences[entry.ShardId] = entry.Sequence + 1; } - - if (_mutationLogStore != null && wireEntry != null) - { - _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); - } } private long NextMutationLogSequence(string shardId) @@ -1481,7 +1540,7 @@ namespace GameCult.Networking { // R-Q: the log is the durable record of every commit, replica-applied ones included, so // the last-write sequence a restart resumes with is rebuilt from it exactly as - // AppendMutationLogEntry/the replica apply paths maintain it live - a row's ordinal for + // LogPrimaryChange/the replica apply paths maintain it live - a row's ordinal for // the selection evaluator's order and cursor must not reset to "never written" on restart. var entries = _mutationLogStore.Read(shard.ShardId); var highest = 0L; @@ -1499,6 +1558,8 @@ namespace GameCult.Networking } } + // Sequences at or below the compaction point are spent even when the log holds none of them. + highest = Math.Max(highest, _mutationLogStore.GetCompactedThrough(shard.ShardId)); if (highest > 0) { _nextLogSequences[shard.ShardId] = highest + 1; @@ -1570,17 +1631,11 @@ namespace GameCult.Networking $"Shard '{shard.ShardId}' log entry {entry.Sequence} has a removed kind with a put payload."); } - var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = HasPrediction(predictionKey); - if (reconcilesPrediction) - { - kind = CultNetDatabaseChangeKind.Reconciled; - } - // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that - // admits the change; the stream handler does both. + // admits the change; the journal does both. A retry after a refused append is a fresh instance, so it matches + // its door again and re-records the entry. var document = _documents.DeserializeRawDocument(message.Document); - using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = reconcilesPrediction ? predictionKey : null })) + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } diff --git a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs index 5b3a9c32..6325a0a0 100644 --- a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs +++ b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs @@ -10,6 +10,11 @@ namespace GameCult.Networking /// /// Stores accepted shard-log entries in their replica catch-up wire shape. /// + /// + /// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of + /// the cache. A store must not call the cache, or wait for a thread that does. Appends must throw rather than + /// swallow a failure: the database burns the sequence and compacts past it, and tells the writer. + /// public interface ICultNetShardMutationLogStore { /// diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs new file mode 100644 index 00000000..c0251af5 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -0,0 +1,456 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // The database's shard log is written by a cache journal, under the cache gate, in cache order. A primary that + // cannot log a committed change burns that change's sequence and compacts past it, so the log can lag the cache but + // never lead it, and every replica behind the hole resynchronizes by snapshot. + public sealed class CultNetJournalTests + { + private const string ShardId = "log-shard"; + + private static readonly CultRecordKey One = new("tests:journal:one"); + private static readonly CultRecordKey Two = new("tests:journal:two"); + private static readonly CultRecordKey Three = new("tests:journal:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetShardDescriptor Shard(CultCache cache, bool primary) => + new(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)]); + + private static CultNetDatabase Database(CultCache cache, bool primary, FlakyLogStore? store) => + new(cache, new CultNetDatabaseOptions { Shards = [Shard(cache, primary)], MutationLogStore = store }); + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + private static List Record(CultNetDatabase database) + { + var seen = new List(); + database.WatchAllChanges().Subscribe(change => seen.Add(KeyOfChange(change))); + return seen; + } + + private static long[] Sequences(FlakyLogStore store) => store.Read(ShardId).Select(entry => entry.Sequence).ToArray(); + + // A store whose refusals and side effects the test scripts. One shard per store. + private sealed class FlakyLogStore : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compactedThrough; + + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public Action? OnRead { get; set; } + public Action? OnAppend { get; set; } + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + OnRead?.Invoke(); + lock (_gate) + { + var entries = _entries.Values.Where(entry => entry.Sequence > afterSequence); + return (limit.HasValue ? entries.Take(limit.Value) : entries).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + OnAppend?.Invoke(); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) + throw new IOException($"The log refused sequence {entry.Sequence}."); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) + { + lock (_gate) + return _compactedThrough; + } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) + throw new IOException("The log refused to compact."); + _compactedThrough = Math.Max(_compactedThrough, sequence); + foreach (var stale in _entries.Keys.Where(key => key <= sequence).ToArray()) + _entries.Remove(stale); + } + } + } + + // The primary as a replica meets it: the database server's own log answer, and the database's own snapshot. + private sealed class PrimaryPeer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _databaseServer; + private readonly CultNetDatabase _database; + + public PrimaryPeer(CultCache cache, CultNetDatabase database) + { + _database = database; + _server = new Server(cache, ServerSecurityOptions.Development()); + _databaseServer = new CultNetDatabaseServer(_server, database); + } + + public CultNetShardLogResponseMessage Log(long afterSequence) => + _databaseServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { + MessageId = "pull", + ShardId = ShardId, + ShardEpoch = 1, + AfterSequence = afterSequence + }); + + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => + Task.FromResult(Log(afterSequence)); + + public Task FetchAsync(CultNetShardDescriptor shard) => + Task.FromResult(_database.CreateShardSnapshotResponse(_database.Shards[0], "snapshot")); + + public void Dispose() + { + _databaseServer.Dispose(); + _server.Dispose(); + } + } + + private static CultNetShardReplicator Replicator(CultNetDatabase replica, PrimaryPeer peer) => + new(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + + private static string? TextOf(CultCache cache, CultRecordKey key) => cache.Get(key)?.Text; + + [Test] + public async Task ARefusedPrimaryAppendPublishesTheChangeTheWriterHearsOfItAndAReplicaBehindItResyncs() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(1)); + var published = Record(primary); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.ShardId, Is.EqualTo(ShardId)); + Assert.That(failure.Sequence, Is.EqualTo(2)); + Assert.That(failure, Is.Not.InstanceOf(), "a committed write must not read as a refused one"); + Assert.That(published, Is.EqualTo(new[] { Two.Value }), "the change was published before the writer heard"); + Assert.That(TextOf(cache, Two), Is.EqualTo("two"), "the commit stands"); + Assert.That(primary.LastWriteSequence(SchemaId(cache), Two), Is.EqualTo(2L), "the row was written at the burned sequence"); + await primary.PutAsync(Three, Note("three")); + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "sequence 2 stays a hole below the compaction point"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3), "a later pull is a quiet no-op"); + } + + [Test] + public async Task ARestartAfterARefusedTailAppendReusesNoSequence() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using (var before = new PrimaryPeer(cache, primary)) + { + await primary.PutAsync(One, Note("one")); + using var replicator = Replicator(replica, before); + await replicator.PullOnceAsync(ShardId); + } + + Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two"))); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "the burned sequence was not minted again"); + using var after = new PrimaryPeer(cache, restarted); + using var resumed = Replicator(replica, after); + await resumed.PullOnceAsync(ShardId); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test] + public async Task AStoreThatRefusesTheAppendAndTheCompactionStillCompactsInMemoryAndNamesBothFailures() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.InnerException, Is.InstanceOf()); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2)); + Assert.That(primary.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2), "the primary's own floor covers a hole its store never compacted"); + Assert.That(primary.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the floor"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two")); + } + + [Test] + public async Task CompactingAndRestartingResumesTheNextSequenceAboveTheCompactionPoint() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + store.CompactThrough(ShardId, 2); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the compaction point"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L })); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(3)); + } + + [Test] + public async Task ARefusedReplicaAppendRecordsNothingAdvancesNoCursorAndTheRetryConverges() + { + var refusals = 0; + var replicaStore = new FlakyLogStore { RefuseAppend = sequence => sequence == 1 && refusals++ == 0 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + using var replicator = Replicator(replica, peer); + + Assert.ThrowsAsync(async () => await replicator.PullOnceAsync(ShardId)); + + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(0)); + Assert.That(replica.GetMutationLog(ShardId), Is.Empty, "memory never holds what the durable log refused"); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L })); + } + + [Test] + public async Task AReentrantShardLogApplyFromAHandlerKeepsTheReplicaLogInPrimaryOrder() + { + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + await primary.PutAsync(Three, Note("three")); + var response = new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = primary.GetMutationLogMessages(ShardId).ToArray() + }; + var replicaStore = new FlakyLogStore(); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + var nested = false; + replica.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + { + if (nested) return; + nested = true; + replica.ApplyShardLogResponseAsync(response).GetAwaiter().GetResult(); + }); + + await replica.ApplyShardLogResponseAsync(response); + + Assert.That(nested, Is.True); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + } + + [Test] + public async Task AReentrantDatabaseWriteFromAHandlerIsLoggedWithItsContextAndPublishedOnce() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var message = database.Documents.CreateRawDocumentPutMessage( + "reentrant-two", new CultRecordHandle(Two), Note("two")); + database.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + database.ApplyPutAsync(message).GetAwaiter().GetResult()); + var published = Record(database); + + await database.PutAsync(One, Note("one")); + + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(store.Read(ShardId).Single(entry => entry.Sequence == 2).Put!.MessageId, Is.EqualTo("reentrant-two"), + "the nested write's own wire message is what the log stored"); + } + + [Test] + public async Task ALogStoreThatWritesTheCacheFailsThroughTheReentryGuardAndTheAdmissionStands() + { + var cache = new CultCache(); + var store = new FlakyLogStore { OnAppend = () => cache.UpsertAsync(Note("from the store"), new CultRecordHandle(Three)).GetAwaiter().GetResult() }; + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.ThrowsAsync(async () => await database.PutAsync(One, Note("one")))!; + + Assert.That(failure.InnerException, Is.InstanceOf(), "the cache's guard refused the store's write"); + Assert.That(TextOf(cache, One), Is.EqualTo("one")); + Assert.That(TextOf(cache, Three), Is.Null); + Assert.That(published, Is.EqualTo(new[] { One.Value })); + } + + [Test] + public void SeveralRefusedAppendsInOneAdmissionAreAllPublishedAndAllReachTheWriter() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + Assert.That(failure.InnerExceptions.OfType().Select(exception => exception.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2)); + } + + [Test] + public async Task APullOnAPrimaryIsLoggedSoAReplicaConverges() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-pull-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("seed"), new CultRecordHandle(One)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var primary = Database(cache, primary: true, new FlakyLogStore()); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + + await writer.UpsertAsync(Note("pulled"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await replicator.PullOnceAsync(ShardId); + + Assert.That(primary.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Does.Contain(Two.Value)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("pulled")); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task AStoreAttachedAfterTheDatabaseExistsHasItsHydrationLogged() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-attach-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("one"), new CultRecordHandle(One)); + await writer.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore()); + + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EquivalentTo(new[] { One.Value, Two.Value })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ASnapshotReadsItsLogSequenceBeforeItsDocumentsSoARacingWriteConverges() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + store.OnRead = () => + { + store.OnRead = null; + primary.PutAsync(Two, Note("two")).GetAwaiter().GetResult(); + }; + + var snapshot = primary.CreateShardSnapshotResponse(primary.Shards[0], "snapshot"); + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], snapshot); + await replicator.PullOnceAsync(ShardId); + + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two"), "a write racing the snapshot reaches the replica by one road or the other"); + } + + [Test] + public async Task ASnapshotAppliedToAPrimaryShardLogsNothing() + { + var cache = new CultCache(); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(One)); + var database = Database(cache, primary: true, new FlakyLogStore()); + var put = database.Documents.CreateRawDocumentPutMessage("snapshot-two", new CultRecordHandle(Two), Note("two")); + + await database.ApplyShardSnapshotResponseAsync(database.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(TextOf(cache, One), Is.Null); + Assert.That(TextOf(cache, Two), Is.EqualTo("two")); + Assert.That(database.GetMutationLog(ShardId), Is.Empty, "the snapshot's removal and its put are both a resync, not a commit"); + } + } +} From 3fb2613abf603d4b5aac138f074f8ef945fd3ef1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:52:14 +0200 Subject: [PATCH 08/26] Networking tests: order is data, so pin each change's own publication, not its delivery order The salvaged S1 and in-place re-upsert tests also asserted the delivery order of the rejected drainer. Under Q-P1 A delivery order across threads and a nested write's depth-first delivery are not guaranteed; each change is published once with its own context, and wire peers drop stale changes by Sequence (Cut 6). Both assertions now say exactly that. The journal tests' reentrant-write recorder subscribes before the handler that writes. Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Networking.Tests/CultNetJournalTests.cs | 2 +- .../CultNetPublicationOrderTests.cs | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index c0251af5..d8d0ebd6 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -303,9 +303,9 @@ namespace GameCult.Networking.Tests var database = Database(cache, primary: true, store); var message = database.Documents.CreateRawDocumentPutMessage( "reentrant-two", new CultRecordHandle(Two), Note("two")); + var published = Record(database); database.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => database.ApplyPutAsync(message).GetAwaiter().GetResult()); - var published = Record(database); await database.PutAsync(One, Note("one")); diff --git a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs index 65e290a4..62d31e69 100644 --- a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs @@ -67,7 +67,9 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B"); Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write"); Assert.That(replicaCache.Get(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary"); - Assert.That(seen.Last(), Is.EqualTo("B"), "a subscriber's final state for the key"); + // Delivery is not ordered across threads (order is data: a wire peer drops the stale change by Sequence, Cut 6), + // so a subscriber sees both changes, once each, in either order. + Assert.That(seen, Is.EquivalentTo(new[] { "A", "B" }), "each change is published exactly once"); }); } @@ -336,7 +338,9 @@ namespace GameCult.Networking.Tests await database.PutPredictedAsync(key, Note("predicted")); await database.ApplyPutAsync(message); - Assert.That(seen, Is.EqualTo(new (CultNetDatabaseChangeKind, string?)[] + // The observer's nested write is delivered depth-first, inside the delivery of the change that caused it, so + // the database may publish the two in either order; what is pinned is that each carries its own context. + Assert.That(seen, Is.EquivalentTo(new (CultNetDatabaseChangeKind, string?)[] { (CultNetDatabaseChangeKind.Predicted, null), (CultNetDatabaseChangeKind.Reconciled, "predicted"), From af7751e26395fd007e7df4f1bb08a0f167227333 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:24:40 +0200 Subject: [PATCH 09/26] Journal tests: the nested shard-log apply must finish, and the store-writes-cache probe fires once The reentrant apply test now asserts the nested call completes without throwing: a re-recorded sequence used to be swallowed by the subscriber's handler and hide a non-replacing in-memory log. The store-writes-cache probe writes once, so a cache that let the journal re-enter it fails the test instead of recursing until the host crashes. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetJournalTests.cs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index d8d0ebd6..25d9b5cf 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -280,16 +280,27 @@ namespace GameCult.Networking.Tests var replicaCache = new CultCache(); var replica = Database(replicaCache, primary: false, replicaStore); var nested = false; + long nestedApplied = 0; + Exception? nestedFailure = null; replica.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => { if (nested) return; nested = true; - replica.ApplyShardLogResponseAsync(response).GetAwaiter().GetResult(); + try + { + nestedApplied = replica.ApplyShardLogResponseAsync(response).GetAwaiter().GetResult(); + } + catch (Exception exception) + { + nestedFailure = exception; + } }); await replica.ApplyShardLogResponseAsync(response); Assert.That(nested, Is.True); + Assert.That(nestedFailure, Is.Null, "the nested apply re-records entries the outer apply already logged"); + Assert.That(nestedApplied, Is.EqualTo(3)); Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L, 3L })); Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); @@ -319,7 +330,12 @@ namespace GameCult.Networking.Tests public async Task ALogStoreThatWritesTheCacheFailsThroughTheReentryGuardAndTheAdmissionStands() { var cache = new CultCache(); - var store = new FlakyLogStore { OnAppend = () => cache.UpsertAsync(Note("from the store"), new CultRecordHandle(Three)).GetAwaiter().GetResult() }; + var store = new FlakyLogStore(); + store.OnAppend = () => + { + store.OnAppend = null; + cache.UpsertAsync(Note("from the store"), new CultRecordHandle(Three)).GetAwaiter().GetResult(); + }; var database = Database(cache, primary: true, store); var published = Record(database); From e79f3ee3707a06cf75a0c2a390752bc74bd795fa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:52:02 +0200 Subject: [PATCH 10/26] Journal tests: pin the survivors Stryker found in the changed journal code Restart over a retained log resumes above its highest sequence and reads its latest from the durable log; put and delete doors log their own wire messages and change kinds; a reconciled put is logged as an update; a replica's watermark follows the primary's sequences; an admission racing disposal logs nothing. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetJournalTests.cs | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 25d9b5cf..02fa159e 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -304,6 +304,7 @@ namespace GameCult.Networking.Tests Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L, 3L })); Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + Assert.That(replica.CurrentAsOf(ShardId), Is.EqualTo(3UL), "the replica's own watermark follows the primary's sequences"); } [Test] @@ -468,5 +469,103 @@ namespace GameCult.Networking.Tests Assert.That(TextOf(cache, Two), Is.EqualTo("two")); Assert.That(database.GetMutationLog(ShardId), Is.Empty, "the snapshot's removal and its put are both a resync, not a commit"); } + + [Test] + public async Task ARestartOverARetainedLogResumesAboveItsHighestSequence() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "a restarted database reads its latest sequence from the durable log"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L, 2L, 3L })); + } + + [Test] + public async Task APutAndDeleteDoorLogTheirOwnWireMessagesAndChangeKinds() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var added = database.Documents.CreateRawDocumentPutMessage("put-added", new CultRecordHandle(One), Note("one")); + var updated = database.Documents.CreateRawDocumentPutMessage("put-updated", new CultRecordHandle(One), Note("uno")); + + await database.ApplyPutAsync(added); + await database.ApplyPutAsync(updated); + await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage { MessageId = "delete-one", SchemaId = SchemaId(cache), RecordKey = One.Value }); + + var entries = store.Read(ShardId); + Assert.That(entries.Select(entry => entry.ChangeKind), Is.EqualTo(new[] { "added", "updated", "removed" })); + Assert.That(entries[0].Put!.MessageId, Is.EqualTo("put-added")); + Assert.That(entries[1].Put!.MessageId, Is.EqualTo("put-updated")); + Assert.That(entries[2].Delete!.MessageId, Is.EqualTo("delete-one")); + Assert.That(DateTimeOffset.TryParse(entries[0].CommittedAt, out _), Is.True, "a primary stamps its entries"); + } + + [Test] + public async Task AReconciledPutIsLoggedAsTheUpdateItIs() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(One), Note("authoritative")); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + var published = new List(); + database.Watch().Subscribe(change => published.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("predicted")); + await database.ApplyPutAsync(message); + + Assert.That(published, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(database.GetMutationLog(shard.ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated }), + "a prediction is never logged and its reconciliation is an ordinary update"); + } + + [Test] + public async Task AnAdmissionRacingDisposalLogsNothingAfterTheDatabaseIsDisposed() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var entered = new System.Threading.ManualResetEventSlim(); + CultNetDatabase? database = null; + // Registered before the database, so it runs first inside the writer's hold, while another thread disposes the database. + using var holdUntilDisposed = cache.AddJournal(_ => + { + entered.Set(); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + try + { + database!.GetLatestMutationLogSequence(ShardId); + } + catch (ObjectDisposedException) + { + return; + } + + System.Threading.Thread.Sleep(1); + } + }); + database = Database(cache, primary: true, store); + var writer = Task.Run(() => database.PutAsync(One, Note("one"))); + Assert.That(entered.Wait(TimeSpan.FromSeconds(5)), Is.True); + + database.Dispose(); + await writer; + + Assert.That(Sequences(store), Is.Empty, "the admission was journaled after disposal began and must not reach the durable log"); + } } } From 13072fc5d77ae46fa17fe214525f8dcfa876e156 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 01:08:03 +0200 Subject: [PATCH 11/26] CultNetDatabase: a door's wire entry carries only its message The change kind in the door's wire entry was restated from the log entry by NormalizeWireLogEntry, so computing it was dead code (Stryker survivor). The entry now carries just the door's put or delete message. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 202f88b5..91fe6ef9 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1252,17 +1252,12 @@ namespace GameCult.Networking change.Key, change.Document, change.PreviousDocument); + // A door's own wire message is what the log stores (its kind, sequence and shard are restated from the entry). var wireEntry = _mutationLogStore == null ? null - : door?.Put != null - ? new CultNetShardLogEntryMessage - { - ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated", - Put = door.Put - } - : door?.Delete != null - ? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete } - : ToLogEntryMessage(entry); + : door is { Put: not null } or { Delete: not null } + ? new CultNetShardLogEntryMessage { Put = door.Put, Delete = door.Delete } + : ToLogEntryMessage(entry); RecordMutationLogEntry(entry, wireEntry); } catch (Exception cause) From 49582cf43ec6eac83a0bbb42538a3e1a73b37276 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:38:50 +0200 Subject: [PATCH 12/26] Soul probes: commit the pass-3 P-tests and the Cut 4 probes as recovered SoulPub3Tests.cs is from 1ccc4927 (recovered pass-3 P-tests); SoulP4ProbeTests.cs is from 8ae8b3dc. Committed as Soul wrote them; the Finding-category probes fail on this commit and are the tests for the fixes that follow. Co-Authored-By: Claude Sonnet 5.5 --- .../SoulP4ProbeTests.cs | 304 ++++++++++++++++++ .../SoulPub3Tests.cs | 276 ++++++++++++++++ 2 files changed, 580 insertions(+) create mode 100644 tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs create mode 100644 tests/GameCult.Networking.Tests/SoulPub3Tests.cs diff --git a/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs new file mode 100644 index 00000000..2377a02b --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass on Cut 4 (scratch; not for merge). Category Finding asserts the invariant Soul expects and is expected to + // fail on 13072fc5; Pin asserts a promise that should hold; Measure reports numbers. + public sealed class SoulP4ProbeTests + { + private const string ShardId = "soul-shard"; + private static readonly CultRecordKey One = new("soul:p4:one"); + private static readonly CultRecordKey Two = new("soul:p4:two"); + private static readonly CultRecordKey Three = new("soul:p4:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + private sealed class Store : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compacted; + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public int AppendDelayMs { get; set; } + public ManualResetEventSlim InAppend { get; } = new(); + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + lock (_gate) + { + var e = _entries.Values.Where(x => x.Sequence > afterSequence); + return (limit.HasValue ? e.Take(limit.Value) : e).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + InAppend.Set(); + if (AppendDelayMs > 0) Thread.Sleep(AppendDelayMs); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) throw new IOException($"refused {entry.Sequence}"); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) { lock (_gate) return _compacted; } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) throw new IOException("refused compact"); + _compacted = Math.Max(_compacted, sequence); + foreach (var k in _entries.Keys.Where(k => k <= sequence).ToArray()) _entries.Remove(k); + } + } + } + + private sealed class Peer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _dbServer; + private readonly CultNetDatabase _db; + public Peer(CultCache cache, CultNetDatabase db) + { + _db = db; + _server = new Server(cache, ServerSecurityOptions.Development()); + _dbServer = new CultNetDatabaseServer(_server, db); + } + public CultNetShardLogResponseMessage Log(long after) => _dbServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { MessageId = "pull", ShardId = ShardId, ShardEpoch = 1, AfterSequence = after }); + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => Task.FromResult(Log(afterSequence)); + public Task FetchAsync(CultNetShardDescriptor shard) => Task.FromResult(_db.CreateShardSnapshotResponse(_db.Shards[0], "snap")); + public void Dispose() { _dbServer.Dispose(); _server.Dispose(); } + } + + private static int StashCount(CultNetDatabase db) + { + var field = typeof(CultNetDatabase).GetField("_stash", BindingFlags.NonPublic | BindingFlags.Instance)!; + var stash = (System.Collections.ICollection)field.GetValue(db)!; + lock (typeof(CultNetDatabase).GetField("_logGate", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(db)!) + return stash.Count; + } + + // F1: the recorded residual is "tail reuse". A hole in the middle of the log after a double refusal and a restart + // strands a replica behind it for good: the restarted primary has no floor, the store serves S+1 after S-1. + [Test, Category("SoulFinding")] + public async Task F1_MidLogHoleAfterDoubleRefusalAndRestartStrandsAReplica() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + await primary.PutAsync(One, Note("one")); + using (var p0 = new Peer(cache, primary)) + using (var r0 = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = p0, SnapshotFetcher = p0 })) + await r0.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + store.RefuseCompact = false; + await primary.PutAsync(Three, Note("three")); + primary.Dispose(); + + var restarted = Db(cache, true, store); + using var peer = new Peer(cache, restarted); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + var errors = new List(); + for (var attempt = 0; attempt < 3; attempt++) + { + try { await replicator.PullOnceAsync(ShardId); } + catch (Exception e) { errors.Add(e.GetType().Name + ": " + e.Message); } + } + + TestContext.Out.WriteLine($"store={string.Join(",", store.Read(ShardId).Select(e => e.Sequence))} floor={restarted.GetCompactedMutationLogSequence(ShardId)} errors=[{string.Join(" | ", errors)}]"); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + // F2: Q-P5 A says the writer gets a typed CultNetShardLogException. A commit of two changes on a refusing store + // hands the writer an AggregateException; a writer's catch (CultNetShardLogException) misses it. + [Test, Category("SoulFinding")] + public void F2_ABatchOnARefusingStoreReachesTheWriterAsTheTypedException() + { + var store = new Store { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Db(cache, true, store); + Exception? caught = null; + try + { + cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }); + } + catch (Exception e) { caught = e; } + TestContext.Out.WriteLine($"writer caught {caught?.GetType().FullName}"); + Assert.That(caught, Is.InstanceOf()); + } + + // F3: main publishes PutPredictedAsync as Predicted and records the prediction whatever instance it is given. The + // shape rule refuses a door whose instance is already the cached one, so a mutate-then-predict is published as + // Updated, is logged on a primary, and its reconciliation is published as Updated. + [Test, Category("SoulFinding")] + public async Task F3_APredictionOfTheCachedInstanceIsPublishedAsAPrediction() + { + var cache = new CultCache(); + var db = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = db.Shards[0]; + var kinds = new List(); + db.Watch().Subscribe(c => kinds.Add(c.Kind)); + await db.PutPredictedAsync(One, Note("p1")); + var held = cache.Get(One)!; + held.Text = "p2"; + await db.PutPredictedAsync(One, held); + var auth = db.Documents.CreateRawDocumentPutMessage("a", new CultRecordHandle(One), Note("auth")); + auth.ShardId = shard.ShardId; + auth.ShardEpoch = shard.Epoch; + await db.ApplyPutAsync(auth); + TestContext.Out.WriteLine($"kinds=[{string.Join(",", kinds)}] log=[{string.Join(",", db.GetMutationLog(shard.ShardId).Select(e => e.Kind))}]"); + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(db.GetMutationLog(shard.ShardId).Select(e => e.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + // F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an + // unrelated key waits for the whole append. + [Test, Category("SoulFinding")] + public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend() + { + var store = new Store { AppendDelayMs = 400 }; + var cache = new CultCache(); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + var db = Db(cache, true, store); + var writer = Task.Run(() => cache.UpsertAsync(Note("one"), new CultRecordHandle(One)).GetAwaiter().GetResult()); + Assert.That(store.InAppend.Wait(5000), Is.True); + var clock = Stopwatch.StartNew(); + _ = cache.Get(Two); + var waited = clock.Elapsed.TotalMilliseconds; + await writer; + TestContext.Out.WriteLine($"reader of an unrelated key waited {waited:0.0} ms for a 400 ms append"); + Assert.That(waited, Is.LessThan(100)); + } + + // P1: the constructor registers the observer before the journal, so a change stashed by the journal always finds + // its publisher, even while another thread writes the cache during construction. + [Test, Category("SoulPin")] + public void P1_ConstructionUnderConcurrentWritesOrphansNoStash() + { + var orphans = 0; + for (var round = 0; round < 200; round++) + { + var cache = new CultCache(); + using var stop = new CancellationTokenSource(); + var writer = Task.Run(() => + { + var i = 0; + while (!stop.IsCancellationRequested) + cache.UpsertAsync(Note("w" + i++), new CultRecordHandle(new CultRecordKey("soul:p4:race:" + (i % 4)))).GetAwaiter().GetResult(); + }); + Thread.SpinWait(200 * (round % 7)); + var db = Db(cache, true, null); + Thread.SpinWait(2000); + stop.Cancel(); + writer.GetAwaiter().GetResult(); + orphans += StashCount(db); + db.Dispose(); + } + + Assert.That(orphans, Is.EqualTo(0), "no stash entry outlives its admission"); + } + + // P2: every stash entry is removed by its publication. + [Test, Category("SoulPin")] + public async Task P2_TheStashDrainsAfterEveryWrite() + { + var cache = new CultCache(); + var db = Db(cache, true, new Store()); + await db.PutAsync(One, Note("one")); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + cache.Commit(batch => batch.Upsert(Note("three"), new CultRecordHandle(Three))); + await db.DeleteAsync(One); + Assert.That(StashCount(db), Is.EqualTo(0)); + } + + // P3: a replica never applies an entry past a hole. The primary burns 2 with compaction refused, then logs 3; a + // replica at 1 is answered compacted_history and resyncs, never applying 3 on top of 1. + [Test, Category("SoulPin")] + public async Task P3_AReplicaAtTheHoleResyncsRatherThanSkippingIt() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + using var peer = new Peer(cache, primary); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + await primary.PutAsync(Three, Note("three")); + Assert.That(peer.Log(1).Reason, Is.EqualTo("compacted_history")); + var applied = await replicator.PullOnceAsync(ShardId); + Assert.That(applied, Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test, Category("SoulMeasure")] + public async Task M1_BareWriteLatencyWithTheFileLogStore() + { + var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N")); + try + { + var cache = new CultCache(); + _ = Db(cache, true, new CultNetFileShardMutationLogStore(root)); + var times = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await cache.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + times.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"file store bare write ms: first100 avg {times.Take(100).Average():0.000}, 400-500 avg {times.Skip(400).Take(100).Average():0.000}, last100 avg {times.Skip(900).Average():0.000}, max {times.Max():0.0}"); + var bare = new CultCache(); + var t2 = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await bare.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + t2.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"no database bare write ms: last100 avg {t2.Skip(900).Average():0.000}"); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, true); + } + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulPub3Tests.cs b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs new file mode 100644 index 00000000..dacfd470 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs @@ -0,0 +1,276 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + [CultDocument("tests.soul_throwing_name", "tests.soul_throwing_name.v1")] + [MessagePack.MessagePackObject] + public sealed class SoulThrowingName + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultName] + public string Name + { + get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text; + set { } + } + } + + [NonParallelizable] + public sealed class SoulPub3Tests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId => CultDocumentRegistry.Shared.GetRequired().SchemaId; + + // Another thread becomes the drainer and parks inside a subscriber on the change "slow". + private static (Task Writer, ManualResetEventSlim Release) BlockDrainer(CultNetDatabase database, CultCache cache, out IDisposable subscription) + { + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + subscription = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "slow") + { + entered.Set(); + release.Wait(Wait); + } + }); + var writer = Task.Run(() => cache.UpsertAsync(Note("slow"), new CultRecordHandle(new CultRecordKey("soul3:slow"))).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True, "drainer parked"); + return (writer, release); + } + + // P1: a write that returns has been logged and published. + [Test] + public async Task P1_AWriteThatReturnedIsLoggedAndPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue<(string? Text, int Thread)>(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue((change.Document?.Text, Environment.CurrentManagedThreadId))); + var (writer, release) = BlockDrainer(database, cache, out var block); + var fast = new CultRecordKey("soul3:fast"); + var writerThread = 0; + await Task.Run(async () => { writerThread = Environment.CurrentManagedThreadId; await database.PutAsync(fast, Note("fast")); }); + + var logged = database.LastWriteSequence(SchemaId, fast); + var published = seen.Any(entry => entry.Text == "fast"); + release.Set(); + await writer; + block.Dispose(); + var deliveredOn = seen.First(entry => entry.Text == "fast").Thread; + TestContext.Progress.WriteLine($"SOUL P1 logged-at-return={logged} published-at-return={published} writer={writerThread} deliveredOn={deliveredOn}"); + Assert.Multiple(() => + { + Assert.That(logged, Is.Not.Null, "PutAsync returned before its change was logged"); + Assert.That(published, Is.True, "PutAsync returned before its change was published"); + }); + } + + // P2: a subscriber that waits (bounded) for another thread's write to be observed. + [Test] + public async Task P2_ASubscriberWaitingForAnotherThreadsWriteIsNotStarved() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var other = new ManualResetEventSlim(); + var observedInTime = false; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "other") + other.Set(); + if (change.Document?.Text == "trigger") + { + Task.Run(() => database.PutAsync(new CultRecordKey("soul3:other"), Note("other"))); + observedInTime = other.Wait(TimeSpan.FromSeconds(2)); + } + }); + await database.PutAsync(new CultRecordKey("soul3:trigger"), Note("trigger")); + Assert.That(observedInTime, Is.True, "another thread's committed write never reached subscribers while this handler ran"); + } + + // P3: one observer's escaped exception (fail-fast R3 handler) loses one change; it must not stall all later ones. + [Test] + public async Task P3_AnEscapedObserverExceptionDoesNotStallLaterChanges() + { + var previous = ObservableSystem.GetUnhandledExceptionHandler(); + ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); + try + { + var cache = new CultCache(); + using var thrower = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom"); + }); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try { await cache.UpsertAsync(Note("boom"), new CultRecordHandle(new CultRecordKey("soul3:boom"))); } + catch (InvalidOperationException) { } + + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + var after = new CultRecordKey("soul3:after"); + await database.PutAsync(after, Note("after")); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + finally + { + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + } + } + + // P4: an admission that throws inside Apply after minting a Sequence (a [CultName] getter) must not stall later changes. + [Test] + public async Task P4_AThrowMidApplyDoesNotStallLaterChanges() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try + { + cache.Commit(batch => + { + batch.Upsert(new SoulThrowingName { Text = "ok" }, new CultRecordHandle(new CultRecordKey("soul3:ok"))); + batch.Upsert(new SoulThrowingName { Text = "boom" }, new CultRecordHandle(new CultRecordKey("soul3:boomname"))); + }); + } + catch (InvalidOperationException) { } + + var after = new CultRecordKey("soul3:after-apply"); + await database.PutAsync(after, Note("after")); + TestContext.Progress.WriteLine($"SOUL P4 ok-in-cache={cache.Get(new CultRecordKey("soul3:ok")) != null}"); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + + // P5: writers no longer carry their own delivery; the backlog behind one drainer grows without bound. + [Test] + public async Task P5_OtherWritersDoNotPileAnUnboundedBacklogOnOneDrainer() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var delivered = 0; + var backlogAtBDone = -1; + const int bWrites = 2000; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text?.StartsWith("b") == true) Interlocked.Increment(ref delivered); + if (change.Document?.Text == "a") + { + Task.Run(async () => + { + for (var i = 0; i < bWrites; i++) + await database.PutAsync(new CultRecordKey($"soul3:b:{i}"), Note($"b{i}")); + }).Wait(Wait); + backlogAtBDone = bWrites - Volatile.Read(ref delivered); + } + }); + await database.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + TestContext.Progress.WriteLine($"SOUL P5 undelivered after B's {bWrites} writes returned: {backlogAtBDone}"); + Assert.That(backlogAtBDone, Is.EqualTo(0)); + } + + // P6: a prediction still queued when the authoritative put arrives. + [Test] + public async Task P6_AnAuthoritativePutArrivingBeforeItsPredictionIsDeliveredStillReconciles() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul3:predict"); + var kinds = new ConcurrentQueue(); + using var watch = database.WatchRecord(key).Subscribe(change => kinds.Enqueue(change.Kind)); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + var (writer, release) = BlockDrainer(database, cache, out var block); + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + release.Set(); + await writer; + block.Dispose(); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + // P7: a replica's absent-key delete is logged ahead of an earlier entry still queued; a bounded pull then sees a gap. + [Test] + public async Task P7_AReplicaLogStaysInSequenceOrderWhenAnAbsentDeleteOvertakesAQueuedPut() + { + const string shardId = "chain3"; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [SchemaId])] + }); + var p = Make(new CultCache(), true); + var rCache = new CultCache(); + var r = Make(rCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to, int? limit = null) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId), limit).ToArray() + }); + } + + var z = new CultRecordKey("soul3:z"); + await p.PutAsync(z, Note("z")); + await Pull(p, r); + await Pull(r, c); + rCache.Remove(z); + await p.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + await p.DeleteAsync(z); + + var (writer, release) = BlockDrainer(r, rCache, out var block); + await Pull(p, r); + release.Set(); + await writer; + block.Dispose(); + + TestContext.Progress.WriteLine($"SOUL P7 r log order: {string.Join(",", r.GetMutationLog(shardId).Select(e => e.Sequence))}"); + Assert.Multiple(() => + { + Assert.That(r.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.DoesNotThrowAsync(() => Pull(r, c, limit: 1), "a bounded pull from R"); + }); + } + } +} From 2eb0e8d6a6097d02506540208dd6a17e8c6f1906 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:48:49 +0200 Subject: [PATCH 13/26] Journal fixes: hole floor on restart, one log exception per call, no shape guard, no stray shard-0 logging F1: InitializeLogSequencesFromStore raises the in-memory floor to the highest gap above the store's compaction point, so a restart after a double refusal or an interrupted compaction answers compacted_history instead of a gapped log. F2: the journal raises one CultNetShardLogException per call carrying every burned (shard, sequence); the unwrap rule for a cache-aggregated hold is documented on the exception and on the cache commit. F3: TakeDoor no longer refuses a door whose instance is the cached one, so mutate-then-predict publishes Predicted and is not logged. F6: a door-less change no shard matches is not logged; doors keep the fallback shard they were resolved to. Dispose disposes the journal before the observer. The log store doc says a slow store blocks cache readers. Soul F4 is ignored as an accepted follow-up, M1 is explicit. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 8 +- src/GameCult.Networking/CultNetDatabase.cs | 97 ++++++--- .../CultNetShardMutationLogStorage.cs | 2 +- .../CultNetJournalTests.cs | 204 +++++++++++++++++- .../SoulP4ProbeTests.cs | 4 +- 5 files changed, 276 insertions(+), 39 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index e8ec8fd6..68fbfb1b 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2124,7 +2124,10 @@ namespace GameCult.Caching Remove(handle.Key); } - // false: a condition failed; nothing was written, changed in memory, or published. + // false: a condition failed; nothing was written, changed in memory, or published. A commit that succeeds and then + // fails in a journal or observer still stands and is published; the writer receives the failure. One failure arrives + // as itself, several as an AggregateException. A writer that wants a particular failure (CultNetShardLogException, when + // a database's log refused the commit) catches that type, or unwraps AggregateException.InnerExceptions. public bool Commit(Action stage) { return Land(stage, wait: true) == CultCommitOutcome.Committed; @@ -2427,7 +2430,8 @@ namespace GameCult.Caching // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not // guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate, // in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even - // if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown. + // if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their + // origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions. // If the body threw, its exception wins. internal T Held(Func body) { diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 91fe6ef9..bcfb1ca4 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -464,33 +464,39 @@ namespace GameCult.Networking } /// - /// Raised to the writer when a shard primary committed a change and could not log it. The commit stands and the - /// change was published; the shard's log burned and compacted past it, so every replica - /// behind it resynchronizes from a snapshot. It is deliberately not an : - /// callers that read that type as a refused write must not mistake a committed one for it. When several things - /// fail in one admission, the writer sees an that contains this one. + /// One shard-log sequence that a primary minted for a committed change and burned because it could not log it. /// + public readonly record struct CultNetBurnedSequence(string ShardId, long Sequence); + + /// + /// Raised to the writer when a shard primary committed changes and could not log them. The commits stand and the + /// changes were published; each shard's log burned the sequences in and compacted past them, + /// so every replica behind one resynchronizes from a snapshot. A single journal call raises one of these carrying + /// every burned sequence, whatever number of changes or shards failed. It is deliberately not an + /// : callers that read that type as a refused write must not mistake a + /// committed one for it. + /// + /// + /// The cache may still wrap this in an when something else failed in the same hold + /// (an observer, a second database's journal). A writer that wants the log failure catches this type, or unwraps + /// and looks for it. is the + /// store's failure, or an of them when several changes failed. + /// public sealed class CultNetShardLogException : Exception { /// /// Creates a shard log exception. /// - public CultNetShardLogException(string shardId, long sequence, Exception cause) - : base($"Shard '{shardId}' committed a change but could not log it; sequence {sequence} is burned and compacted past.", cause) + public CultNetShardLogException(IReadOnlyList burned, Exception cause) + : base($"Committed changes could not be logged; sequences {string.Join(", ", burned.Select(entry => $"{entry.ShardId}:{entry.Sequence}"))} are burned and compacted past.", cause) { - ShardId = shardId; - Sequence = sequence; + Burned = burned; } /// - /// Gets the shard whose log refused the change. + /// Gets every sequence burned by the journal call, in commit order. /// - public string ShardId { get; } - - /// - /// Gets the shard-log sequence that was minted for the change and burned. - /// - public long Sequence { get; } + public IReadOnlyList Burned { get; } } /// @@ -1103,8 +1109,8 @@ namespace GameCult.Networking } _disposed = true; - _cacheChanges.Dispose(); _cacheJournal.Dispose(); + _cacheChanges.Dispose(); lock (_logGate) _stash.Clear(); _changes.Dispose(); @@ -1124,6 +1130,8 @@ namespace GameCult.Networking } List? failures = null; + List? burned = null; + List? burnCauses = null; foreach (var change in changes) { var identity = change.Document ?? change.PreviousDocument; @@ -1158,9 +1166,14 @@ namespace GameCult.Networking { RecordReplicatedEntry(shard, logKind, descriptor.SchemaId, change, replicated); } - else if (door?.NoLog != true && shard.IsPrimary) + else if (door?.NoLog != true && shard.IsPrimary && (door != null || _shards.Any(candidate => candidate.Matches(descriptor, change.Key)))) { - LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door); + // A door chose its shard; a bare write belongs to a shard only when one matches it. + if (LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door) is { } burn) + { + (burned ??= new List()).Add(new CultNetBurnedSequence(shard.ShardId, burn.Sequence)); + (burnCauses ??= new List()).Add(burn.Cause); + } } } catch (Exception exception) @@ -1172,7 +1185,14 @@ namespace GameCult.Networking _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } - // Every change is stashed and will be published; the writer hears of a failed log afterwards. + // Every change is stashed and will be published; the writer hears of a failed log afterwards, in one exception. + if (burned != null) + { + (failures ??= new List()).Insert( + 0, + new CultNetShardLogException(burned, burnCauses!.Count == 1 ? burnCauses[0] : new AggregateException(burnCauses))); + } + if (failures != null) { throw failures.Count == 1 ? failures[0] : new AggregateException(failures); @@ -1228,8 +1248,8 @@ namespace GameCult.Networking // A primary mints the sequence, appends durably, then records in memory. If it cannot log the change, the change is // still committed: the sequence is burned and compacted past, so a replica behind it is told compacted_history and - // resynchronizes by snapshot, and the writer is told with a CultNetShardLogException. - private void LogPrimaryChange( + // resynchronizes by snapshot. The burn is returned for the journal to tell the writer in one CultNetShardLogException. + private (long Sequence, Exception Cause)? LogPrimaryChange( CultNetShardDescriptor shard, CultNetDatabaseChangeKind logKind, string schemaId, @@ -1259,6 +1279,7 @@ namespace GameCult.Networking ? new CultNetShardLogEntryMessage { Put = door.Put, Delete = door.Delete } : ToLogEntryMessage(entry); RecordMutationLogEntry(entry, wireEntry); + return null; } catch (Exception cause) { @@ -1272,15 +1293,15 @@ namespace GameCult.Networking cause = new AggregateException(cause, compactCause); } - throw new CultNetShardLogException(shard.ShardId, sequence, cause); + return (sequence, cause); } } } - // A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had - // held, so its change carries that instance as the new document and never as the previous one; a removal door - // removes an instance, so its change carries it as the previous document and no new one. A plain removal of an - // instance a put door admitted, or an observer re-upserting that instance in place, is neither, and gets no context. + // A door's context belongs to one specific change, recognised by shape: a put door's change carries the instance as + // its new document; a removal door removes an instance, so its change carries it as the previous document and no + // new one. A plain removal of an instance a put door admitted is neither, and gets no context. A door's own change + // is journaled first, in its own hold, and takes its context there. private DoorContext? TakeDoor(CultCacheDocumentChange change) { var identity = change.Document ?? change.PreviousDocument; @@ -1291,7 +1312,7 @@ namespace GameCult.Networking var matches = change.Document == null ? door.Removal - : !door.Removal && !ReferenceEquals(change.PreviousDocument, change.Document); + : !door.Removal; return matches && ((ICollection>)_doors) .Remove(new KeyValuePair(identity, door)) ? door @@ -1537,10 +1558,21 @@ namespace GameCult.Networking // the last-write sequence a restart resumes with is rebuilt from it exactly as // LogPrimaryChange/the replica apply paths maintain it live - a row's ordinal for // the selection evaluator's order and cursor must not reset to "never written" on restart. - var entries = _mutationLogStore.Read(shard.ShardId); + var entries = _mutationLogStore.Read(shard.ShardId).OrderBy(entry => entry.Sequence).ToArray(); + var compacted = _mutationLogStore.GetCompactedThrough(shard.ShardId); var highest = 0L; + var expected = compacted + 1; + var hole = 0L; foreach (var entry in entries) { + // A sequence the log lacks between two it holds was burned (its append and its compaction both failed, or + // a compaction was interrupted). Replicas must be told compacted_history for it, not served a gapped log. + if (entry.Sequence > compacted && entry.Sequence > expected) + { + hole = entry.Sequence - 1; + } + + expected = Math.Max(expected, entry.Sequence + 1); if (entry.Sequence > highest) { highest = entry.Sequence; @@ -1554,7 +1586,12 @@ namespace GameCult.Networking } // Sequences at or below the compaction point are spent even when the log holds none of them. - highest = Math.Max(highest, _mutationLogStore.GetCompactedThrough(shard.ShardId)); + highest = Math.Max(highest, compacted); + if (hole > 0) + { + _burnedThrough[shard.ShardId] = hole; + } + if (highest > 0) { _nextLogSequences[shard.ShardId] = highest + 1; diff --git a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs index 6325a0a0..4c93e000 100644 --- a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs +++ b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs @@ -12,7 +12,7 @@ namespace GameCult.Networking /// /// /// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of - /// the cache. A store must not call the cache, or wait for a thread that does. Appends must throw rather than + /// the cache and blocks its readers too, because reads take the same gate. A store must not call the cache, or wait for a thread that does. Appends must throw rather than /// swallow a failure: the database burns the sequence and compacts past it, and tells the writer. /// public interface ICultNetShardMutationLogStore diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 02fa159e..208feb1d 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; @@ -153,8 +154,7 @@ namespace GameCult.Networking.Tests var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; - Assert.That(failure.ShardId, Is.EqualTo(ShardId)); - Assert.That(failure.Sequence, Is.EqualTo(2)); + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 2) })); Assert.That(failure, Is.Not.InstanceOf(), "a committed write must not read as a refused one"); Assert.That(published, Is.EqualTo(new[] { Two.Value }), "the change was published before the writer heard"); Assert.That(TextOf(cache, Two), Is.EqualTo("two"), "the commit stands"); @@ -356,13 +356,14 @@ namespace GameCult.Networking.Tests var database = Database(cache, primary: true, store); var published = Record(database); - var failure = Assert.Throws(() => cache.Commit(batch => + var failure = Assert.Throws(() => cache.Commit(batch => { batch.Upsert(Note("one"), new CultRecordHandle(One)); batch.Upsert(Note("two"), new CultRecordHandle(Two)); }))!; - Assert.That(failure.InnerExceptions.OfType().Select(exception => exception.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 1), new CultNetBurnedSequence(ShardId, 2) }), "one exception carries every burned sequence"); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2), "and every store failure"); Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); Assert.That(database.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2)); } @@ -567,5 +568,200 @@ namespace GameCult.Networking.Tests Assert.That(Sequences(store), Is.Empty, "the admission was journaled after disposal began and must not reach the durable log"); } + + [Test] + public void ABatchAcrossTwoShardsWhoseStoreRefusesReachesTheWriterAsOneExceptionNamingBothShards() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = + [ + new CultNetShardDescriptor("shard-a", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "a:"), + new CultNetShardDescriptor("shard-b", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "b:") + ], + MutationLogStore = store + }); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("a"), new CultRecordHandle(new CultRecordKey("a:1"))); + batch.Upsert(Note("b"), new CultRecordHandle(new CultRecordKey("b:1"))); + }))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence("shard-a", 1), new CultNetBurnedSequence("shard-b", 1) })); + } + + // The cache aggregates whatever else fails in the same hold; the documented unwrap finds the log failure inside. + [Test] + public void ALogFailureBesideAnotherFailureInTheSameHoldIsFoundByUnwrappingTheAggregate() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Database(cache, primary: true, store); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + var logFailure = failure.InnerExceptions.OfType().Single(); + Assert.That(logFailure.Burned.Select(burned => burned.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(failure.InnerExceptions.OfType().Single().Message, Is.EqualTo("second journal")); + } + + [Test] + public async Task ABareWriteOfASchemaNoShardOwnsIsNeitherLoggedNorReplicated() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "n", Body = "unrelated" }, new CultRecordHandle(new CultRecordKey("unrelated:n"))); + await cache.UpsertAsync(Note("one"), new CultRecordHandle(One)); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the unrelated write minted no sequence"); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); + } + + // A door chose its shard, by the fallback when none matches; the journal honours it. + [Test] + public async Task ADatabaseWriteOfASchemaNoShardOwnsIsLoggedInTheFallbackShard() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + + await database.PutAsync(new CultRecordKey("unrelated:n"), new MeshQuickstartNote { NoteId = "n", Body = "door" }); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L })); + } + + private static async Task ApplyAuthoritativeAsync(CultNetDatabase database, CultRecordKey key, string text) + { + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(key), Note(text)); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + await database.ApplyPutAsync(message); + } + + private static CultNetDatabaseOptions ClientOptions() => new() + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }; + + [Test] + public async Task APredictionOverAPredictionIsStillAPredictionAndTheAuthoritativeWriteReconcilesOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, ClientOptions()); + var kinds = new List(); + database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("p1")); + await database.PutPredictedAsync(One, Note("p2")); + await ApplyAuthoritativeAsync(database, One, "authoritative"); + await ApplyAuthoritativeAsync(database, One, "again"); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + [Test] + public async Task AGameSessionPredictionOfTheInstanceTheCacheAlreadyHoldsIsAPredictionThatIsNeverLogged() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "journal-session", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + using var node = await GameCult.Mesh.CultMesh.CreateNodeAsync(Path.Combine(rootPath, "world.ccmp"), new GameCult.Mesh.CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = ClientOptions() + }); + using var session = GameCult.Mesh.CultMesh.CreateGameSession(node, new GameCult.Mesh.CultMeshGameSessionOptions + { + ServeSimulationObservations = false, + ServeVerseDiscovery = false, + ServePeerExchange = false + }); + var kinds = new List(); + node.Database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await session.PredictAsync(One, Note("p1")); + var held = node.Cache.Get(One)!; + held.Text = "p2"; + await session.PredictAsync(One, held); + await ApplyAuthoritativeAsync(node.Database, One, "authoritative"); + + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(node.Database.GetMutationLog(node.Database.Shards[0].ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + [Test] + public async Task ADatabaseRecordsEveryDeleteEntryItAppliesWhileAnotherThreadRemovesTheSameKeys() + { + const int rounds = 20000; + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + var schemaId = SchemaId(replicaCache); + for (var i = 1; i <= rounds; i++) + await replicaCache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey("race:" + i))); + + var current = 0; + var stop = false; + var remover = Task.Run(() => + { + var next = 1; + while (next <= rounds && !Volatile.Read(ref stop)) + { + if (Volatile.Read(ref current) >= next) + { + replicaCache.Remove(new CultRecordKey("race:" + next)); + next++; + } + } + }); + for (var i = 1; i <= rounds; i++) + { + Volatile.Write(ref current, i); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage + { + Sequence = i, + CommittedAt = DateTimeOffset.UtcNow.ToString("O"), + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d" + i, + SchemaId = schemaId, + RecordKey = "race:" + i, + ShardId = ShardId, + ShardEpoch = 1 + } + } + ] + }); + } + + Volatile.Write(ref stop, true); + await remover; + + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(Enumerable.Range(1, rounds).Select(i => (long)i)), + "a replica's log holds every entry of the primary's, whoever removed the key first"); + } } } diff --git a/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs index 2377a02b..feb2cb3e 100644 --- a/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs +++ b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs @@ -187,7 +187,7 @@ namespace GameCult.Networking.Tests // F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an // unrelated key waits for the whole append. - [Test, Category("SoulFinding")] + [Test, Category("SoulFinding"), Ignore("Accepted follow-up F4: a log append runs under the cache gate, so readers wait; documented on ICultNetShardMutationLogStore.")] public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend() { var store = new Store { AppendDelayMs = 400 }; @@ -267,7 +267,7 @@ namespace GameCult.Networking.Tests Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); } - [Test, Category("SoulMeasure")] + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion; writes a log file.")] public async Task M1_BareWriteLatencyWithTheFileLogStore() { var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N")); From f28ca3a89f2793387dc8a55a98f6dbb869ed57d6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:49:50 +0200 Subject: [PATCH 14/26] CultNetBurnedSequence: a plain struct, since netstandard2.1 has no IsExternalInit The record struct in 2eb0e8d6 did not compile for the library's netstandard2.1 target. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index bcfb1ca4..97d822a5 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -466,7 +466,39 @@ namespace GameCult.Networking /// /// One shard-log sequence that a primary minted for a committed change and burned because it could not log it. /// - public readonly record struct CultNetBurnedSequence(string ShardId, long Sequence); + public readonly struct CultNetBurnedSequence : IEquatable + { + /// + /// Creates a burned sequence. + /// + public CultNetBurnedSequence(string shardId, long sequence) + { + ShardId = shardId; + Sequence = sequence; + } + + /// + /// Gets the shard whose log burned the sequence. + /// + public string ShardId { get; } + + /// + /// Gets the burned shard-log sequence. + /// + public long Sequence { get; } + + /// + public bool Equals(CultNetBurnedSequence other) => Sequence == other.Sequence && string.Equals(ShardId, other.ShardId, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is CultNetBurnedSequence other && Equals(other); + + /// + public override int GetHashCode() => (StringComparer.Ordinal.GetHashCode(ShardId ?? string.Empty) * 397) ^ Sequence.GetHashCode(); + + /// + public override string ToString() => $"{ShardId}:{Sequence}"; + } /// /// Raised to the writer when a shard primary committed changes and could not log them. The commits stand and the From f3076264fc2c36f749764c12b1c872b5d8f12a51 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:51:14 +0200 Subject: [PATCH 15/26] Journal tests: a plain put of an unowned schema is unlogged, a door put keeps the fallback shard Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetJournalTests.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 208feb1d..b50d675d 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -627,15 +627,23 @@ namespace GameCult.Networking.Tests Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); } - // A door chose its shard, by the fallback when none matches; the journal honours it. + // A door chose its shard, by the fallback when none matches; the journal honours it. A plain PutAsync carries no + // door, so an unowned schema written through it is cached and not logged. [Test] - public async Task ADatabaseWriteOfASchemaNoShardOwnsIsLoggedInTheFallbackShard() + public async Task AnAuthoritativePutOfASchemaNoShardOwnsIsLoggedInTheFallbackShardButAPlainPutIsNot() { var store = new FlakyLogStore(); var cache = new CultCache(); var database = Database(cache, primary: true, store); + var shard = database.Shards[0]; - await database.PutAsync(new CultRecordKey("unrelated:n"), new MeshQuickstartNote { NoteId = "n", Body = "door" }); + await database.PutAsync(new CultRecordKey("unrelated:plain"), new MeshQuickstartNote { NoteId = "p", Body = "plain" }); + Assert.That(Sequences(store), Is.Empty); + + var message = database.Documents.CreateRawDocumentPutMessage("door", new CultRecordHandle(new CultRecordKey("unrelated:door")), new MeshQuickstartNote { NoteId = "d", Body = "door" }); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + await database.ApplyPutAsync(message); Assert.That(Sequences(store), Is.EqualTo(new[] { 1L })); } From 9f6b2d40818506cb1182f55e2426f967144f98ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 04:08:45 +0200 Subject: [PATCH 16/26] CultNetDatabase: a write no shard owns is refused, not cached and never logged PutAsync and DeleteAsync carry no door, so the journal cannot place an unowned schema and now skips it; accepting the write anyway was a silent loss of replication. They resolve their shard strictly and throw CultNetUnownedSchemaException (an InvalidOperationException naming the schema) before the cache is touched. Doors keep the fallback shard they were resolved to. No GameCult consumer configures shards outside vendored CultLib copies: AetheriaEve's node uses the default PrimaryAll shard, which owns everything. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 38 +++++++++++++++++-- .../CultNetJournalTests.cs | 16 ++++++-- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 97d822a5..4ef5820b 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -500,6 +500,32 @@ namespace GameCult.Networking public override string ToString() => $"{ShardId}:{Sequence}"; } + /// + /// Raised when a database write names a schema and key that no configured shard owns. Nothing was cached. + /// + public sealed class CultNetUnownedSchemaException : InvalidOperationException + { + /// + /// Creates an unowned schema exception. + /// + public CultNetUnownedSchemaException(string schemaId, CultRecordKey key) + : base($"No shard owns schema '{schemaId}' key '{key.Value}'; the write was refused.") + { + SchemaId = schemaId; + Key = key; + } + + /// + /// Gets the schema that no shard owns. + /// + public string SchemaId { get; } + + /// + /// Gets the key of the refused write. + /// + public CultRecordKey Key { get; } + } + /// /// Raised to the writer when a shard primary committed changes and could not log them. The commits stand and the /// changes were published; each shard's log burned the sequences in and compacted past them, @@ -931,8 +957,7 @@ namespace GameCult.Networking if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(document.GetType()); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); + EnsurePrimary(OwningShard(descriptor, key), descriptor.SchemaId, key); return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } @@ -968,8 +993,7 @@ namespace GameCult.Networking { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); + EnsurePrimary(OwningShard(descriptor, key), descriptor.SchemaId, key); _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; @@ -1908,6 +1932,12 @@ namespace GameCult.Networking return _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) ?? _shards[0]; } + // The shard that owns the schema and key, for a write that carries no door and so cannot be placed by the wire message: + // a write no shard owns would be cached and never logged, so it is refused. + private CultNetShardDescriptor OwningShard(CultDocumentDescriptor descriptor, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) + ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); + private static void EnsurePrimary(CultNetShardDescriptor shard, string schemaId, CultRecordKey key, long? expectedEpoch = null) { if (expectedEpoch.HasValue && expectedEpoch.Value != shard.Epoch) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index b50d675d..806ad08b 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -627,17 +627,25 @@ namespace GameCult.Networking.Tests Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); } - // A door chose its shard, by the fallback when none matches; the journal honours it. A plain PutAsync carries no - // door, so an unowned schema written through it is cached and not logged. + // The database is the sharded write door: a write it cannot place in a shard is refused before anything is cached. + // (An authoritative wire put carries a door that names its shard, and keeps the fallback shard.) [Test] - public async Task AnAuthoritativePutOfASchemaNoShardOwnsIsLoggedInTheFallbackShardButAPlainPutIsNot() + public async Task ADatabaseWriteOfASchemaNoShardOwnsIsRefusedNamingTheSchemaAndAnAuthoritativePutKeepsTheFallbackShard() { var store = new FlakyLogStore(); var cache = new CultCache(); var database = Database(cache, primary: true, store); var shard = database.Shards[0]; + var unowned = cache.Registry.GetRequired().SchemaId; + var plainKey = new CultRecordKey("unrelated:plain"); - await database.PutAsync(new CultRecordKey("unrelated:plain"), new MeshQuickstartNote { NoteId = "p", Body = "plain" }); + var put = Assert.ThrowsAsync(async () => + await database.PutAsync(plainKey, new MeshQuickstartNote { NoteId = "p", Body = "plain" }))!; + Assert.That(put.SchemaId, Is.EqualTo(unowned)); + Assert.That(put.Message, Does.Contain(unowned)); + Assert.That(put, Is.InstanceOf(), "a refused write reads as one"); + Assert.That(cache.Get(plainKey), Is.Null, "nothing was cached"); + Assert.ThrowsAsync(async () => await database.DeleteAsync(plainKey)); Assert.That(Sequences(store), Is.Empty); var message = database.Documents.CreateRawDocumentPutMessage("door", new CultRecordHandle(new CultRecordKey("unrelated:door")), new MeshQuickstartNote { NoteId = "d", Body = "door" }); From e0075217b7e6a6102b03cd132eb34b85d047a852 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 04:21:48 +0200 Subject: [PATCH 17/26] Networking test: the outside-the-shard fixture is written to the cache, since the database refuses it Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Networking.Tests/NetworkingTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index d9ee9704..1c849fd7 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -4212,13 +4212,13 @@ namespace GameCult.Networking.Tests Schema = "tests.networking_note.v1", Text = "two" }); - // Outside the shard's key prefix: must reach the non-shard snapshot but never the + // Outside the shard's key prefix (written to the cache directly: the database refuses a write no shard owns): must reach the non-shard snapshot but never the // shard-scoped one, so an unfiltered request still proves shard membership is enforced. - await database.PutAsync(new CultRecordKey("other-note:outside-shard"), new NetworkSchemaNote + await cache.UpsertAsync(new NetworkSchemaNote { Schema = "tests.networking_note.v1", Text = "outside" - }); + }, new CultRecordHandle(new CultRecordKey("other-note:outside-shard"))); var nonShard = registry.CreateRawSnapshotResponse(cache, "snapshot-nonshard-unfiltered", filter: null); var shardScoped = database.CreateShardSnapshotResponse(shard, "snapshot-shard-unfiltered", filter: null); From e79882a8ec386cd5ecb24a032774b2541c105c60 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 04:31:03 +0200 Subject: [PATCH 18/26] CultNetDatabase: no shard owns it, no write, on every path; the fallback shard is deleted ResolveShardInternal throws CultNetUnownedSchemaException instead of falling back to the first shard. PutAsync, DeleteAsync, PutPredictedAsync, ApplyPutAsync and ApplyDeleteAsync refuse an unowned schema before the cache is touched; a remote put reaches the server's generic catch and is answered as an application rejection like any other refused remote put. ResolveShard throws (documented), CanWriteAuthoritatively returns false, and a bare cache write no shard owns is neither logged nor published. The unused string overload of EnsureClientAuthority is deleted. No GameCult consumer configures shards outside vendored CultLib copies; the default shard owns everything. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 55 +++----- .../CultNetJournalTests.cs | 132 +++++++++++++++--- 2 files changed, 136 insertions(+), 51 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 4ef5820b..083d309f 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -501,7 +501,8 @@ namespace GameCult.Networking } /// - /// Raised when a database write names a schema and key that no configured shard owns. Nothing was cached. + /// Raised when a database write, or a resolution, names a schema and key that no configured shard owns. A refused write + /// caches nothing. /// public sealed class CultNetUnownedSchemaException : InvalidOperationException { @@ -655,7 +656,7 @@ namespace GameCult.Networking { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - return ResolveShardInternal(descriptor, key).IsPrimary; + return FindShard(descriptor, key)?.IsPrimary ?? false; } /// Gets whether this database instance can submit a prediction for the document at a key. @@ -906,6 +907,7 @@ namespace GameCult.Networking /// /// Resolves the shard that governs the supplied schema and key. /// + /// No configured shard owns the schema and key. public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key) { ThrowIfDisposed(); @@ -957,7 +959,7 @@ namespace GameCult.Networking if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(document.GetType()); - EnsurePrimary(OwningShard(descriptor, key), descriptor.SchemaId, key); + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } @@ -993,7 +995,7 @@ namespace GameCult.Networking { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - EnsurePrimary(OwningShard(descriptor, key), descriptor.SchemaId, key); + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; @@ -1199,7 +1201,13 @@ namespace GameCult.Networking var door = TakeDoor(change); var documentType = identity.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); - var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); + var shard = door?.Shard ?? FindShard(descriptor, change.Key); + if (shard == null) + { + // No shard owns a bare cache write of this schema: the database neither logs nor publishes it. + continue; + } + var kind = door?.Kind ?? change.Kind switch { CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, @@ -1222,9 +1230,8 @@ namespace GameCult.Networking { RecordReplicatedEntry(shard, logKind, descriptor.SchemaId, change, replicated); } - else if (door?.NoLog != true && shard.IsPrimary && (door != null || _shards.Any(candidate => candidate.Matches(descriptor, change.Key)))) + else if (door?.NoLog != true && shard.IsPrimary) { - // A door chose its shard; a bare write belongs to a shard only when one matches it. if (LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door) is { } burn) { (burned ??= new List()).Add(new CultNetBurnedSequence(shard.ShardId, burn.Sequence)); @@ -1864,20 +1871,6 @@ namespace GameCult.Networking }; } - private void EnsureClientAuthority(string schemaId, CultRecordKey key) - { - if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, schemaId, key))) - { - return; - } - - var shard = ResolveShardInternal(schemaId, key); - throw new CultNetShardAuthorityException( - shard, - $"Runtime '{_runtimeId}' does not have client prediction authority for schema '{schemaId}' key '{key.Value}'.", - "not_client_authority"); - } - private void EnsureClientAuthority(CultDocumentDescriptor descriptor, CultRecordKey key) { if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, descriptor, key))) @@ -1922,21 +1915,15 @@ namespace GameCult.Networking } } - private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) - { - return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0]; - } + // No shard owns a schema and key that no descriptor matches, and nothing is written to one: every write path refuses. + private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? throw new CultNetUnownedSchemaException(schemaId, key); - private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) - { - return _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) ?? _shards[0]; - } + private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) => + FindShard(descriptor, key) ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); - // The shard that owns the schema and key, for a write that carries no door and so cannot be placed by the wire message: - // a write no shard owns would be cached and never logged, so it is refused. - private CultNetShardDescriptor OwningShard(CultDocumentDescriptor descriptor, CultRecordKey key) => - _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) - ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); + private CultNetShardDescriptor? FindShard(CultDocumentDescriptor descriptor, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)); private static void EnsurePrimary(CultNetShardDescriptor shard, string schemaId, CultRecordKey key, long? expectedEpoch = null) { diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 806ad08b..5e18bcd7 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -614,46 +614,144 @@ namespace GameCult.Networking.Tests } [Test] - public async Task ABareWriteOfASchemaNoShardOwnsIsNeitherLoggedNorReplicated() + public async Task ABareWriteOfASchemaNoShardOwnsIsNeitherLoggedNorPublishedNorReplicated() { var store = new FlakyLogStore(); var cache = new CultCache(); var database = Database(cache, primary: true, store); + var published = Record(database); await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "n", Body = "unrelated" }, new CultRecordHandle(new CultRecordKey("unrelated:n"))); await cache.UpsertAsync(Note("one"), new CultRecordHandle(One)); Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the unrelated write minted no sequence"); Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); + Assert.That(published, Is.EqualTo(new[] { One.Value }), "no shard owns the unrelated write, so the database does not publish it"); } - // The database is the sharded write door: a write it cannot place in a shard is refused before anything is cached. - // (An authoritative wire put carries a door that names its shard, and keeps the fallback shard.) - [Test] - public async Task ADatabaseWriteOfASchemaNoShardOwnsIsRefusedNamingTheSchemaAndAnAuthoritativePutKeepsTheFallbackShard() + // The database is the sharded write door: a schema and key no shard owns cannot be written on any path, and a refused + // write caches and logs nothing. + private static (CultNetDatabase Database, CultCache Cache, FlakyLogStore Store, string SchemaId) OwnedNothingOfNote() { var store = new FlakyLogStore(); var cache = new CultCache(); - var database = Database(cache, primary: true, store); - var shard = database.Shards[0]; - var unowned = cache.Registry.GetRequired().SchemaId; - var plainKey = new CultRecordKey("unrelated:plain"); + return (Database(cache, primary: true, store), cache, store, cache.Registry.GetRequired().SchemaId); + } + + private static readonly CultRecordKey Unowned = new("unrelated:note"); + + private static CultNetDocumentPutRawMessage UnownedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "wire" }); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + [Test] + public void ADatabasePutOrDeleteOfASchemaNoShardOwnsIsRefusedNamingTheSchema() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); var put = Assert.ThrowsAsync(async () => - await database.PutAsync(plainKey, new MeshQuickstartNote { NoteId = "p", Body = "plain" }))!; + await database.PutAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "plain" }))!; Assert.That(put.SchemaId, Is.EqualTo(unowned)); Assert.That(put.Message, Does.Contain(unowned)); Assert.That(put, Is.InstanceOf(), "a refused write reads as one"); - Assert.That(cache.Get(plainKey), Is.Null, "nothing was cached"); - Assert.ThrowsAsync(async () => await database.DeleteAsync(plainKey)); + Assert.That(cache.Get(Unowned), Is.Null, "nothing was cached"); + Assert.ThrowsAsync(async () => await database.DeleteAsync(Unowned)); Assert.That(Sequences(store), Is.Empty); + } - var message = database.Documents.CreateRawDocumentPutMessage("door", new CultRecordHandle(new CultRecordKey("unrelated:door")), new MeshQuickstartNote { NoteId = "d", Body = "door" }); - message.ShardId = shard.ShardId; - message.ShardEpoch = shard.Epoch; - await database.ApplyPutAsync(message); + [Test] + public void ARemoteAuthoritativePutOfASchemaNoShardOwnsIsRefusedAndCachesNothing() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); - Assert.That(Sequences(store), Is.EqualTo(new[] { 1L })); + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(UnownedPut(database)))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Null); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public async Task ARemoteDeleteOfASchemaNoShardOwnsIsRefusedEvenWhenTheKeyIsCached() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + + var refusal = Assert.ThrowsAsync(async () => await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + }))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the refused delete removed nothing"); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public void APredictionOfASchemaNoShardOwnsIsRefused() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + Shards = [new CultNetShardDescriptor("only-notes", "local", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)])], + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + + Assert.ThrowsAsync(async () => + await database.PutPredictedAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "predicted" })); + Assert.That(cache.Get(Unowned), Is.Null); + } + + [Test] + public void ASchemaNoShardOwnsCannotBeWrittenAuthoritativelyAndHasNoShardToResolve() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + + Assert.That(database.CanWriteAuthoritatively(Unowned), Is.False); + Assert.That(database.CanWriteAuthoritatively(One), Is.True, "an owned schema is unaffected"); + var refusal = Assert.Throws(() => database.ResolveShard(unowned, Unowned))!; + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(database.ResolveShard(SchemaId(cache), One).ShardId, Is.EqualTo(ShardId)); + } + + // The server answers a refused remote put the way it answers any other: it logs the failure and sends the peer a + // CultNetErrorMessage carrying the refusal. The handler is reached through its private delegate with an + // uninitialized peer, as the R-AM test does; the send then fails on that peer, after the failure was logged. + [Test] + public async Task TheServerAnswersARemotePutOfASchemaNoShardOwnsWithAnApplicationRejection() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + + try { await handler(UnownedPut(database), peer); } catch (Exception) { } + + Assert.That(logger.Errors, Has.Some.Contains("CultNet raw put failed").And.Contains(unowned)); + Assert.That(cache.Get(Unowned), Is.Null); + } + + private sealed class CapturingLogger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } } private static async Task ApplyAuthoritativeAsync(CultNetDatabase database, CultRecordKey key, string text) From dac712b09c0a3b1931657f12b13bbcf4c948c973 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 05:29:35 +0200 Subject: [PATCH 19/26] CultNetDatabaseServer: a selection skips cached rows no shard owns instead of failing ResolveShard now throws for an unowned schema, so one such cached row would have failed the whole selection. The server passes a row filter (CultNetDatabase.Owns) to CreateSelectionResponse, which gains an optional rowFilter that SelectPage already took, so those rows are not part of what the database serves and the rest is answered. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 2 ++ .../CultNetDatabaseServer.cs | 2 ++ .../CultNetDocumentRegistry.cs | 6 ++++-- .../CultNetJournalTests.cs | 21 +++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 083d309f..07336488 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1922,6 +1922,8 @@ namespace GameCult.Networking private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) => FindShard(descriptor, key) ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); + internal bool Owns(CultDocumentDescriptor descriptor, CultRecordKey key) => FindShard(descriptor, key) != null; + private CultNetShardDescriptor? FindShard(CultDocumentDescriptor descriptor, CultRecordKey key) => _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)); diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index d53315ed..78124acc 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -135,6 +135,8 @@ namespace GameCult.Networking // R-Q: asOf is one shard-log watermark, so a selection whose matched rows span more than // one shard's log is refused rather than answered against a watermark that is not // exact for all of them. + // A row whose schema no shard owns is not part of what this database serves: it is left out, not an error. + rowFilter: _database.Owns, shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId, cursorKey: _database.CursorKey, // S-9: once the matched rows resolve to one shard, asOf is that shard's own watermark, diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index eb23d503..a47f28b3 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -338,6 +338,7 @@ namespace GameCult.Networking /// row's ordinal (the reference: ); /// is the snapshot the page is exact for when the selection is not shard-scoped, or the fallback /// used only when is null or the selection matches no rows (S-9). + /// , when supplied, leaves out the cached rows it rejects before evaluation. /// public CultNetSnapshotResponseRawV1Message CreateSelectionResponse( CultCache cache, @@ -348,13 +349,14 @@ namespace GameCult.Networking CultNetDocumentMessageOptions? options = null, Func? shardIdOf = null, CultNetSelectionCursorKey? cursorKey = null, - Func? asOfForShard = null) + Func? asOfForShard = null, + Func? rowFilter = null) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (selection == null) throw new ArgumentNullException(nameof(selection)); if (ordinalOf == null) throw new ArgumentNullException(nameof(ordinalOf)); - var page = SelectPage(cache, selection, ordinalOf, asOf, options, shardIdOf: shardIdOf, cursorKey: cursorKey, asOfForShard: asOfForShard); + var page = SelectPage(cache, selection, ordinalOf, asOf, options, rowFilter, shardIdOf, cursorKey, asOfForShard); return new CultNetSnapshotResponseRawV1Message { MessageId = RequireNonEmpty(messageId, nameof(messageId)), diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 5e18bcd7..3bbb72ef 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -745,6 +745,27 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(Unowned), Is.Null); } + // One cached row of a schema no shard owns is not part of what this database serves: the selection answers with the + // rest, it does not fail as a whole. + [Test] + public async Task ASelectionSkipsACachedRowNoShardOwnsAndAnswersWithTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSelectionResponse(new CultNetSnapshotRequestV1Message + { + MessageId = "select-all", + Selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document } + }); + + Assert.That(response.Documents!.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + Assert.That(response.Matched, Is.EqualTo(1u)); + } + private sealed class CapturingLogger : GameCult.Logging.ILogger { public List Errors { get; } = new(); From 396cccafd48594df1e8057d5468eb05441b95eee Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 06:09:14 +0200 Subject: [PATCH 20/26] Soul probes: commit the pass 4c probes (Q1-Q5) as recovered from 9c30f67f The Finding-category probes fail on this commit; the fixes that follow are their tests. Co-Authored-By: Claude Sonnet 5.5 --- .../SoulP4cProbeTests.cs | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs diff --git a/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs new file mode 100644 index 00000000..98956d54 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs @@ -0,0 +1,140 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4c scratch probes (not for merge). + public sealed class SoulP4cProbeTests + { + private const string ShardId = "soulc-shard"; + private static readonly CultRecordKey One = new("soulc:one"); + private static readonly CultRecordKey Unowned = new("soulc:unowned"); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private sealed class RefusingStore : ICultNetShardMutationLogStore + { + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) => Array.Empty(); + public void Append(string shardId, CultNetShardLogEntryMessage entry) => throw new IOException("refused " + entry.Sequence); + public long GetCompactedThrough(string shardId) => 0; + public void CompactThrough(string shardId, long sequence) { } + } + + private sealed class Logger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + // Q1: the server's own remote-put path is a writer. A put that committed and could not be logged is answered as a + // failure: the peer is told its write failed, while the cache holds it and it was published. + [Test, Category("SoulFinding")] + public async Task Q1_ServerAnswersACommittedButUnloggedRemotePutAsAFailure() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, new RefusingStore()); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new Logger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + + try { await handler(message, peer); } catch (Exception) { } + + TestContext.WriteLine("errors: " + string.Join(" | ", logger.Errors)); + Assert.That(cache.Get(One)?.Text, Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Has.None.Contains("raw put failed"), "a committed put must not be reported to the peer as a failed one"); + } + + // Q2: reads serve a cached row no shard owns, the database watch never publishes its changes: a view built from + // GetAll + Watch (CultMesh Collection, the subscription server's snapshot + WatchAllChanges) goes stale. + [Test, Category("SoulFinding")] + public async Task Q2_ReadsServeAnUnownedRowWhoseChangesTheWatchNeverPublishes() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var seen = new List(); + database.Watch().Subscribe(change => seen.Add(change.Document?.Body ?? "")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v1" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v2" }, new CultRecordHandle(Unowned)); + + var read = database.GetAll().Select(note => note.Body).ToArray(); + TestContext.WriteLine($"GetAll={string.Join(",", read)} watch={string.Join(",", seen)}"); + Assert.That(read.Length == 0 || seen.Count > 0, Is.True, "a row the database reads out must be one whose changes it publishes"); + } + + // Q3: a replica's log apply door writes a row its shard does not own. The snapshot apply path filters by + // ShardMatchesRawDocument; the log path does not. + [Test, Category("SoulFinding")] + public async Task Q3_AReplicaLogApplyWritesARowItsShardDoesNotOwn() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, store: null); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "r", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + Exception? thrown = null; + try + { + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = DateTimeOffset.UtcNow.ToString("O"), ChangeKind = "added", Put = put }] + }); + } + catch (Exception exception) { thrown = exception; } + + TestContext.WriteLine($"thrown={thrown?.GetType().Name}: {thrown?.Message}; cached={cache.Get(Unowned)?.Body}; log={replica.GetMutationLog(ShardId).Count}"); + Assert.That(cache.Get(Unowned), Is.Null, "no shard owns it here, so no path writes it"); + } + + // Q4 (pin): the unowned refusal is not a shard-authority exception, so the server's routing path does not see it. + [Test, Category("SoulPin")] + public void Q4_TheUnownedRefusalIsNotAShardAuthorityException() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var put = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "w" }); + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(put))!; + Assert.That(refusal, Is.Not.InstanceOf()); + } + + // Q5 (pin): two databases over one cache, both refusing: a writer's catch (CultNetShardLogException) misses. + [Test, Category("SoulPin")] + public void Q5_TwoDatabasesOnOneCacheAggregateTheirLogFailures() + { + var cache = new CultCache(); + _ = Db(cache, primary: true, new RefusingStore()); + _ = Db(cache, primary: true, new RefusingStore()); + var failure = Assert.Throws(() => cache.Commit(batch => + batch.Upsert(Note("x"), new CultRecordHandle(One))))!; + Assert.That(failure.InnerExceptions.OfType().Count(), Is.EqualTo(2)); + } + } +} From 557e70fd036efba5d0a7541106ab125a0c37c7f1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 06:14:49 +0200 Subject: [PATCH 21/26] Cut 4 Soul 4c: committed-unlogged wire writes, database serves only what it owns, typed unowned refusal Q1: HandlePutAsync/HandleDeleteAsync catch a CultNetShardLogException (or an AggregateException holding one) before the generic catch and answer nothing: the write committed and was published, so the peer is not told it was refused. It is logged at warning level. The wire has no committed-with-degraded-log message. Q2: GetAsync/GetAll/GetByName/GetByIndex, the shard-less snapshot, the selection and the subscription snapshot serve only rows a shard owns, so read and watch agree; the CultMesh collections follow through GetAll, and the Mesh untyped variant fallback checks ownership. The raw cache still holds everything. Q3: a replica log apply skips a row its shard does not own, as the snapshot path does, and still records the entry so a chained replica sees no gap. API: rowFilter is internal (public CreateSelectionResponse and CreateRawSnapshotResponse signatures restored, internal overloads added); exception tags on the write methods; Commit's unwrap rule is XML doc; the unowned refusal answers on the wire with code unowned_schema (schema enum in both copies, Rust CultNetErrorCode, CultNetErrorMessage.ForUnownedSchema) and put and delete log it at warning level. Dispose keeps journal-first; a test now fails if the observer goes first. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/cultnet.error.schema.json | 6 +- .../cultnet/cultnet.error.schema.json | 6 +- packages/cultnet-rs/src/contracts.rs | 7 +- src/GameCult.Caching/CultCache.cs | 13 +- src/GameCult.Mesh/CultMesh.cs | 16 +- src/GameCult.Networking/CultNetDatabase.cs | 77 ++++- .../CultNetDatabaseServer.cs | 31 +- .../CultNetDatabaseSubscriptionServer.cs | 3 +- .../CultNetDocumentRegistry.cs | 31 +- .../CultNetSchemaMessages.cs | 13 +- .../CultNetJournalTests.cs | 290 +++++++++++++++++- .../CultNetSchemaContractTests.cs | 11 + 12 files changed, 467 insertions(+), 37 deletions(-) diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index 15880fa1..0d541262 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index 15880fa1..0d541262 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index f31ade85..1808c62f 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -250,8 +250,8 @@ impl CultNetShardDescriptor { } } -/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings -/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth +/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the strings +/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a /// string it does not know refuses to decode rather than guess a meaning for it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -261,6 +261,7 @@ pub enum CultNetErrorCode { CursorInvalid, ReferenceOutsideTarget, VariantUnsupported, + UnownedSchema, } impl CultNetErrorCode { @@ -271,6 +272,7 @@ impl CultNetErrorCode { Self::CursorInvalid => "cursor_invalid", Self::ReferenceOutsideTarget => "reference_outside_target", Self::VariantUnsupported => "variant_unsupported", + Self::UnownedSchema => "unowned_schema", } } @@ -281,6 +283,7 @@ impl CultNetErrorCode { "cursor_invalid" => Ok(Self::CursorInvalid), "reference_outside_target" => Ok(Self::ReferenceOutsideTarget), "variant_unsupported" => Ok(Self::VariantUnsupported), + "unowned_schema" => Ok(Self::UnownedSchema), other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")), } } diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 68fbfb1b..6365261f 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2124,10 +2124,15 @@ namespace GameCult.Caching Remove(handle.Key); } - // false: a condition failed; nothing was written, changed in memory, or published. A commit that succeeds and then - // fails in a journal or observer still stands and is published; the writer receives the failure. One failure arrives - // as itself, several as an AggregateException. A writer that wants a particular failure (CultNetShardLogException, when - // a database's log refused the commit) catches that type, or unwraps AggregateException.InnerExceptions. + /// + /// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published. + /// + /// + /// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer + /// receives the failure. One failure arrives as itself, several as an . A writer that + /// wants a particular failure (a CultNetShardLogException, when a database's log refused the commit) catches that type, + /// or unwraps and looks for it. + /// public bool Commit(Action stage) { return Land(stage, wait: true) == CultCommitOutcome.Committed; diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 52c67482..bdc1dbdc 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -3370,6 +3370,20 @@ namespace GameCult.Mesh $"CultMesh document '{key.Value}' was not found as {typeof(TDocument).FullName}."); } + // A database serves only what its shards own; the cache behind it may hold more. + private static bool DatabaseServes(CultNetDatabase database, object document, CultRecordKey key) + { + try + { + database.ResolveShard(CultDocumentRegistry.Shared.GetRequired(document.GetType()).SchemaId, key); + return true; + } + catch (CultNetUnownedSchemaException) + { + return false; + } + } + private static async Task ReadDatabaseDocumentRequiredAsync( CultNetDatabase database, CultRecordKey key) @@ -3380,7 +3394,7 @@ namespace GameCult.Mesh return document; var untyped = database.Cache.Get(key); - if (untyped != null && IsSameCultDocumentSchema(untyped.GetType())) + if (untyped != null && IsSameCultDocumentSchema(untyped.GetType()) && DatabaseServes(database, untyped, key)) return ConvertUntypedDocument(untyped); throw new KeyNotFoundException( diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 07336488..1cb1a1e0 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -920,16 +920,18 @@ namespace GameCult.Networking public Task GetAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); - return Task.FromResult(_cache.Get(key)); + var document = _cache.Get(key); + return Task.FromResult(document != null && Owns(_cache.Registry.GetRequired(document.GetType()), key) ? document : null); } /// - /// Gets all documents assignable to the requested type. + /// Gets all documents assignable to the requested type. A database serves only what its shards own: a cached row no + /// shard owns is left out here, as it is from every read and projection. The cache itself still holds it. /// public IEnumerable GetAll() where T : class { ThrowIfDisposed(); - return _cache.GetAll(); + return _cache.GetAll().Where(Serves); } /// @@ -938,7 +940,8 @@ namespace GameCult.Networking public T? GetByName(string name) where T : class { ThrowIfDisposed(); - return _cache.GetByName(name); + var document = _cache.GetByName(name); + return document != null && Serves(document) ? document : null; } /// @@ -947,12 +950,15 @@ namespace GameCult.Networking public T? GetByIndex(string alias, string value) where T : class { ThrowIfDisposed(); - return _cache.GetByIndex(alias, value); + var document = _cache.GetByIndex(alias, value); + return document != null && Serves(document) ? document : null; } /// /// Adds or replaces a document at a specific key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task> PutAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); @@ -967,6 +973,7 @@ namespace GameCult.Networking /// /// Applies a locally predicted document for a client-owned input scope. /// + /// No shard owns the schema and key; nothing was written. public async Task> PutPredictedAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); @@ -991,6 +998,8 @@ namespace GameCult.Networking /// /// Deletes a document by key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task DeleteAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); @@ -1004,6 +1013,8 @@ namespace GameCult.Networking /// /// Applies a raw document mutation after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task ApplyPutAsync(CultNetDocumentPutRawMessage message) { ThrowIfDisposed(); @@ -1026,6 +1037,8 @@ namespace GameCult.Networking /// /// Applies a raw document delete after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task ApplyDeleteAsync(CultNetDocumentDeleteMessage message) { ThrowIfDisposed(); @@ -1729,6 +1742,12 @@ namespace GameCult.Networking // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that // admits the change; the journal does both. A retry after a refused append is a fresh instance, so it matches // its door again and re-records the entry. + if (!shard.Matches(descriptor, key)) + { + RecordUnappliedEntry(shard, kind, descriptor.SchemaId, key, entry); + return; + } + var document = _documents.DeserializeRawDocument(message.Document); using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); @@ -1750,6 +1769,12 @@ namespace GameCult.Networking var key = new CultRecordKey(message.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); + if (!shard.Matches(descriptor, key)) + { + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + return; + } + var admitted = false; var previous = _cache.Get(key); if (previous != null) @@ -1761,15 +1786,26 @@ namespace GameCult.Networking if (!admitted) { - // The key is not here, so nothing was admitted to log. This replica's log must still hold every entry of the - // primary's, or a replica chained on it sees a gap. - lock (_logGate) - { - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed, - descriptor.SchemaId, key, document: null, previousDocument: null), entry); - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - } + // The key is not here, so nothing was admitted to log. + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + } + } + + // An entry that changed nothing here (an absent key, or a row this shard does not own - the snapshot path leaves such + // rows out too). This replica's log must still hold every entry of the primary's, or a replica chained on it sees a gap. + private void RecordUnappliedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind kind, + string schemaId, + CultRecordKey key, + CultNetShardLogEntryMessage entry) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, kind, + schemaId, key, document: null, previousDocument: null), entry); + _lastWriteSequence[(schemaId, key.Value)] = entry.Sequence; } } @@ -1924,6 +1960,19 @@ namespace GameCult.Networking internal bool Owns(CultDocumentDescriptor descriptor, CultRecordKey key) => FindShard(descriptor, key) != null; + // Whether the database serves this cached instance: a shard owns its schema and key. A shard that owns everything makes + // the lookup of the instance's key unnecessary. + private bool Serves(object document) + { + if (_shards.Any(shard => shard.SchemaIds.Count == 0 && string.IsNullOrEmpty(shard.KeyPrefix))) + { + return true; + } + + var type = document.GetType(); + return Owns(_cache.Registry.GetRequired(type), GetTrackedKey(document, type)); + } + private CultNetShardDescriptor? FindShard(CultDocumentDescriptor descriptor, CultRecordKey key) => _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)); diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 78124acc..77b03070 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -112,10 +112,13 @@ namespace GameCult.Networking request); } + // A database serves only what its shards own. return _database.Documents.CreateRawSnapshotResponse( _database.Cache, string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId, - request); + request, + options: null, + rowFilter: _database.Owns); } /// @@ -136,6 +139,7 @@ namespace GameCult.Networking // one shard's log is refused rather than answered against a watermark that is not // exact for all of them. // A row whose schema no shard owns is not part of what this database serves: it is left out, not an error. + options: null, rowFilter: _database.Owns, shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId, cursorKey: _database.CursorKey, @@ -314,6 +318,16 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + // The write committed and was published; only its shard log failed, so the peer is not told it was refused. + _server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { _server.Logger.LogError($"CultNet raw put failed: {ex.Message}"); @@ -334,6 +348,15 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + _server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); @@ -606,6 +629,12 @@ namespace GameCult.Networking : messageId; } + // A CultNetShardLogException means the write committed and was published (the writer hears of a failed log afterwards); + // the cache wraps it in an AggregateException when something else failed in the same hold. + private static bool CommittedWithLogFailure(Exception exception) => + exception is CultNetShardLogException || + exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException); + private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception) { return new CultNetErrorMessage diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index f3150740..7ae3c67e 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -421,7 +421,8 @@ namespace GameCult.Networking documentSelection, ordinalOf: (schemaId, key) => _database.LastWriteSequence(schemaId, key) ?? 0, asOf: _database.CurrentAsOf(), - options: null); + options: null, + rowFilter: _database.Owns); // The snapshot is one row per record key by construction, so the duplicate-key throw this // used to guard can no longer fire (docs/cultnet-selection-cut.md, S2-2); deleted rather // than tested around. diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index a47f28b3..62691a3e 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -280,7 +280,16 @@ namespace GameCult.Networking CultCache cache, string messageId, CultNetSnapshotRequestMessage? filter = null, - CultNetDocumentMessageOptions? options = null) + CultNetDocumentMessageOptions? options = null) => + CreateRawSnapshotResponse(cache, messageId, filter, options, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse( + CultCache cache, + string messageId, + CultNetSnapshotRequestMessage? filter, + CultNetDocumentMessageOptions? options, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); @@ -323,7 +332,7 @@ namespace GameCult.Networking ordinalOf = static (_, _) => 0; } - var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options); + var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options, rowFilter); return new CultNetSnapshotResponseRawMessage { @@ -338,7 +347,6 @@ namespace GameCult.Networking /// row's ordinal (the reference: ); /// is the snapshot the page is exact for when the selection is not shard-scoped, or the fallback /// used only when is null or the selection matches no rows (S-9). - /// , when supplied, leaves out the cached rows it rejects before evaluation. /// public CultNetSnapshotResponseRawV1Message CreateSelectionResponse( CultCache cache, @@ -349,8 +357,21 @@ namespace GameCult.Networking CultNetDocumentMessageOptions? options = null, Func? shardIdOf = null, CultNetSelectionCursorKey? cursorKey = null, - Func? asOfForShard = null, - Func? rowFilter = null) + Func? asOfForShard = null) => + CreateSelectionResponse(cache, messageId, selection, ordinalOf, asOf, options, shardIdOf, cursorKey, asOfForShard, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawV1Message CreateSelectionResponse( + CultCache cache, + string messageId, + CultNetSelection selection, + Func ordinalOf, + ulong asOf, + CultNetDocumentMessageOptions? options, + Func? shardIdOf, + CultNetSelectionCursorKey? cursorKey, + Func? asOfForShard, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (selection == null) throw new ArgumentNullException(nameof(selection)); diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 4daa76d5..e7f4cfd9 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -511,7 +511,7 @@ namespace GameCult.Networking [Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; } /// /// Gets or sets the machine-readable refusal code (R-N), e.g. selection_invalid, - /// cursor_stale, cursor_invalid, reference_outside_target. Additive to + /// cursor_stale, cursor_invalid, reference_outside_target, unowned_schema. Additive to /// cultnet.error.v0; a v0 peer that does not know this field ignores it. /// [Key("code")] public string? Code { get; set; } @@ -532,6 +532,17 @@ namespace GameCult.Networking Details = new CultNetErrorDetails { Field = ex.Field, Value = ex.Value } }; + /// + /// Builds the wire error for a write no shard owns: code unowned_schema, details { field: "schemaId", value }. + /// + public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) => + new CultNetErrorMessage + { + Error = $"unowned_schema: {ex.Message}", + Code = "unowned_schema", + Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId } + }; + /// /// Builds the wire error for a cursor refusal (R-N): code cursor_stale with details /// { asOf, current }, or code cursor_invalid with no details. diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 3bbb72ef..7ba70dc8 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -741,7 +741,8 @@ namespace GameCult.Networking.Tests try { await handler(UnownedPut(database), peer); } catch (Exception) { } - Assert.That(logger.Errors, Has.Some.Contains("CultNet raw put failed").And.Contains(unowned)); + Assert.That(logger.Warnings, Has.Some.Contains("raw put refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty, "a refusal is not a fault"); Assert.That(cache.Get(Unowned), Is.Null); } @@ -766,11 +767,296 @@ namespace GameCult.Networking.Tests Assert.That(response.Matched, Is.EqualTo(1u)); } + [CultDocument("tests.journal_indexed", "tests.journal_indexed.v1")] + [MessagePack.MessagePackObject] + public sealed class JournalIndexedNote + { + [MessagePack.Key(0)] + [CultName] + public string Name { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultIndex("tag")] + public string Tag { get; set; } = string.Empty; + } + + // A database serves only what its shards own: reads, the legacy and shard-less snapshot, selections, subscription + // snapshots and the watch agree. The cache behind it still holds the row. + [Test] + public async Task ADatabaseReadsServeOnlyWhatItsShardsOwnWhileTheCacheStillHoldsTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + var seen = Record(database); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new JournalIndexedNote { Name = "n", Tag = "t" }, new CultRecordHandle(new CultRecordKey("unrelated:indexed"))); + + Assert.That(database.GetAll(), Is.Empty); + Assert.That(database.GetAll().Count(), Is.EqualTo(1), "only the owned row"); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(await database.GetAsync(Unowned), Is.Null); + Assert.That(await database.GetAsync(One), Is.Not.Null); + Assert.That(database.GetByName("u"), Is.Null); + Assert.That(database.GetByName("n"), Is.Null); + Assert.That(database.GetByIndex("tag", "t"), Is.Null); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the cache itself still holds it"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "what the database reads out is what it publishes"); + } + + [Test] + public async Task ADatabaseCollectionOfASchemaNoShardOwnsIsEmpty() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + + var collection = GameCult.Mesh.CultMesh.Collection(database); + + Assert.That(await collection.LatestAsync(), Is.Empty); + } + + [Test] + public async Task TheShardlessSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + + Assert.That(response.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + + [Test] + public async Task ASubscriptionSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + var socket = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Dgram, System.Net.Sockets.ProtocolType.Udp); + socket.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "owned-reads-server", Socket = socket }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("owned-reads-client"); + CultNetSnapshotResponseRawMessage? snapshot = null; + client.OnCultNet(message => snapshot = message); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + for (var attempt = 0; attempt < 300 && !client.Connected; attempt++) + await Task.Delay(10); + client.SendCultNet(new CultNetDatabaseSubscribeMessage { MessageId = "subscribe-all", SubscriptionId = "all", IncludeSnapshot = true }); + for (var attempt = 0; attempt < 300 && snapshot == null; attempt++) + await Task.Delay(10); + + Assert.That(snapshot, Is.Not.Null, "the subscription answered a snapshot"); + Assert.That(snapshot!.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + + // A replica applies the log the way it applies a snapshot: a row its shard does not own is left out. The entry is + // still recorded, so a replica chained on this one sees no gap. + [Test] + public async Task AReplicaLogApplySkipsARowItsShardDoesNotOwnButRecordsTheEntry() + { + var cache = new CultCache(); + var replica = Database(cache, primary: false, store: null); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey("unrelated:put")), new MeshQuickstartNote { NoteId = "p", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + var owned = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("owned")); + owned.ShardId = ShardId; + owned.ShardEpoch = 1; + var now = DateTimeOffset.UtcNow.ToString("O"); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = now, ChangeKind = "added", Put = put }, + new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = cache.Registry.GetRequired().SchemaId, + RecordKey = Unowned.Value, + ShardId = ShardId, + ShardEpoch = 1 + } + }, + new CultNetShardLogEntryMessage { Sequence = 3, CommittedAt = now, ChangeKind = "added", Put = owned } + ] + }); + + Assert.That(cache.Get(new CultRecordKey("unrelated:put")), Is.Null, "the put was left out"); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the delete removed nothing the shard owns"); + Assert.That(TextOf(cache, One), Is.EqualTo("owned"), "owned entries still apply"); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + } + + // The server's put and delete handlers are reached through their private delegates with an uninitialized peer (as + // the R-AM test does): anything the handler tries to send throws on that peer, so a handler that returns quietly + // sent nothing. + private static (CultNetDatabaseServer Server, Func Put, Func Delete, CapturingLogger Logger, Server Host) WireHandlers(CultNetDatabase database, CultCache cache) + { + var host = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + host.Logger = logger; + var databaseServer = new CultNetDatabaseServer(host, database); + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + const System.Reflection.BindingFlags flags = System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance; + var put = (Func)typeof(CultNetDatabaseServer).GetField("_putHandler", flags)!.GetValue(databaseServer)!; + var delete = (Func)typeof(CultNetDatabaseServer).GetField("_deleteHandler", flags)!.GetValue(databaseServer)!; + return (databaseServer, message => put(message, peer), message => delete(message, peer), logger, host); + } + + private static CultNetDocumentPutRawMessage OwnedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + return message; + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedBesideAnotherFailureIsStillNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire")); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemoteDeleteWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }); + await database.PutAsync(One, Note("one")); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = SchemaId(cache), + RecordKey = One.Value, + ShardId = ShardId, + ShardEpoch = 1 + }); + + Assert.That(TextOf(cache, One), Is.Null, "the delete committed"); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public void ARemoteDeleteOfASchemaNoShardOwnsIsRefusedOnTheWireAtWarningLevel() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + })); + + Assert.That(logger.Warnings, Has.Some.Contains("raw delete refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty); + } + + // The journal is disposed before the observer: an admission the journal logs while the database is being disposed is + // still published, never logged and then dropped. + [Test] + public async Task ADatabaseDisposedWhileAWriteIsInItsLogAppendStillPublishesWhatItLogged() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var seen = Record(database); + using var inAppend = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + store.OnAppend = () => + { + inAppend.Set(); + release.Wait(TimeSpan.FromSeconds(10)); + }; + var writer = Task.Run(async () => await database.PutAsync(One, Note("one"))); + Assert.That(inAppend.Wait(TimeSpan.FromSeconds(10)), Is.True); + + var dispose = Task.Run(database.Dispose); + await Task.Delay(300); + release.Set(); + await writer; + await dispose; + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the write was logged"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "and published"); + } + private sealed class CapturingLogger : GameCult.Logging.ILogger { public List Errors { get; } = new(); + public List Warnings { get; } = new(); public void LogInfo(string message) { } - public void LogWarning(string message) { } + public void LogWarning(string message) => Warnings.Add(message); public void LogError(string message) => Errors.Add(message); public void LogDebug(string message) { } } diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index b5f4f08f..3aa2705d 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -195,6 +195,17 @@ namespace GameCult.Networking.Tests AssertErrorMatchesSchema(CultNetErrorMessage.ForReferenceOutsideTarget(ex)); } + [Test] + public void ErrorMessage_UnownedSchemaRealBytes_MatchesItsSchema() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + + Assert.That(message.Code, Is.EqualTo("unowned_schema")); + Assert.That(message.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(message.Details.Value, Is.EqualTo("some.schema.v1")); + AssertErrorMatchesSchema(message); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); From 79f144d9eccea2b3377a4a7233e3432fff35ecec Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 06:16:57 +0200 Subject: [PATCH 22/26] CultNetDatabase.Dispose publishes what a writer logged and had not yet published The dispose test showed journal-first alone still lost the change: Dispose won the race against the writer's publication and cleared the stash. Dispose now drains the stash through the publisher after the journal is disposed (so nothing is stashed afterwards) and before the subject completes. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 17 ++++++++++++++++- .../CultNetJournalTests.cs | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 1cb1a1e0..bedf91c6 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1180,10 +1180,22 @@ namespace GameCult.Networking } _disposed = true; + // The journal first: disposing it waits for a journal in flight, so nothing is stashed after this line. What a + // writer has logged and not yet published is then published here, so a change the log holds is never dropped. _cacheJournal.Dispose(); _cacheChanges.Dispose(); + Publication[] unpublished; lock (_logGate) + { + unpublished = _stash.OrderBy(entry => entry.Key).Select(entry => entry.Value).ToArray(); _stash.Clear(); + } + + foreach (var publication in unpublished) + { + Publish(publication); + } + _changes.Dispose(); } @@ -1288,6 +1300,10 @@ namespace GameCult.Networking } } + Publish(publication); + } + + private void Publish(Publication publication) => PublishUntyped( publication.DocumentType, publication.Kind, @@ -1296,7 +1312,6 @@ namespace GameCult.Networking publication.Shard, publication.Document, publication.PreviousDocument); - } private void RecordReplicatedEntry( CultNetShardDescriptor shard, diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 7ba70dc8..d6f9d6d6 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -1022,8 +1022,8 @@ namespace GameCult.Networking.Tests Assert.That(logger.Errors, Is.Empty); } - // The journal is disposed before the observer: an admission the journal logs while the database is being disposed is - // still published, never logged and then dropped. + // A change the log holds is published, however Dispose interleaves with the write: Dispose waits for a journal in + // flight (disposing the journal first) and publishes whatever the writer had logged and not yet published. [Test] public async Task ADatabaseDisposedWhileAWriteIsInItsLogAppendStillPublishesWhatItLogged() { From 6e406879be748537e013f60467d72a2e0ad9bcb7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 06:23:09 +0200 Subject: [PATCH 23/26] Tests: the Mesh same-schema read serves only owned rows; the unowned put answer must try to send Co-Authored-By: Claude Sonnet 5.5 --- .../CultMeshStreamingTests.cs | 26 +++++++++++++++++++ .../CultNetJournalTests.cs | 5 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs index acf5eb6c..63747c29 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs @@ -938,6 +938,32 @@ public sealed class CultMeshStreamingTests await act.Should().ThrowAsync(); } + // A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type + // out of the cache behind it. + [TestCase("tests.mesh_note.v1", true)] + [TestCase("tests.unrelated.v1", false)] + public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served) + { + var cache = new CultCache(); + var key = new CultRecordKey("note:one"); + await cache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cached" }, new CultRecordHandle(key)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "reader", + Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) } + }); + var handle = CultMesh.Document(database, key, CultMesh.Verse("starbridge", "reader")); + + if (served) + { + (await handle.LatestAsync()).Text.Should().Be("cached"); + } + else + { + await FluentActions.Awaiting(() => handle.LatestAsync()).Should().ThrowAsync(); + } + } + [Test] public async Task DocumentHandle_SubmitsPredictionsThroughCultNetDatabase() { diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index d6f9d6d6..8d487bbb 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -727,7 +727,7 @@ namespace GameCult.Networking.Tests // CultNetErrorMessage carrying the refusal. The handler is reached through its private delegate with an // uninitialized peer, as the R-AM test does; the send then fails on that peer, after the failure was logged. [Test] - public async Task TheServerAnswersARemotePutOfASchemaNoShardOwnsWithAnApplicationRejection() + public void TheServerAnswersARemotePutOfASchemaNoShardOwnsWithAnApplicationRejection() { var (database, cache, _, unowned) = OwnedNothingOfNote(); using var server = new Server(cache, ServerSecurityOptions.Development()); @@ -739,7 +739,8 @@ namespace GameCult.Networking.Tests .GetValue(databaseServer)!; var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); - try { await handler(UnownedPut(database), peer); } catch (Exception) { } + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await handler(UnownedPut(database), peer)); Assert.That(logger.Warnings, Has.Some.Contains("raw put refused").And.Contains(unowned)); Assert.That(logger.Errors, Is.Empty, "a refusal is not a fault"); From 29eaabc4ff40efe6088193f65d9b77850002b7b8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 07:03:15 +0200 Subject: [PATCH 24/26] Dispose test: hold the writer in an earlier observer so Dispose must publish what it logged Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetJournalTests.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs index 8d487bbb..6285d674 100644 --- a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -1024,12 +1024,20 @@ namespace GameCult.Networking.Tests } // A change the log holds is published, however Dispose interleaves with the write: Dispose waits for a journal in - // flight (disposing the journal first) and publishes whatever the writer had logged and not yet published. + // flight (disposing the journal first) and publishes whatever the writer had logged and not yet published. The writer + // is held in an earlier cache observer, so Dispose finishes before the writer reaches the database's own observer. [Test] public async Task ADatabaseDisposedWhileAWriteIsInItsLogAppendStillPublishesWhatItLogged() { var store = new FlakyLogStore(); var cache = new CultCache(); + using var inObserver = new ManualResetEventSlim(); + using var releaseObserver = new ManualResetEventSlim(); + using var holdWriter = cache.Watch().Subscribe(_ => + { + inObserver.Set(); + releaseObserver.Wait(TimeSpan.FromSeconds(10)); + }); var database = Database(cache, primary: true, store); var seen = Record(database); using var inAppend = new ManualResetEventSlim(); @@ -1045,8 +1053,10 @@ namespace GameCult.Networking.Tests var dispose = Task.Run(database.Dispose); await Task.Delay(300); release.Set(); - await writer; + Assert.That(inObserver.Wait(TimeSpan.FromSeconds(10)), Is.True, "the writer left the gate and is held in the earlier observer"); await dispose; + releaseObserver.Set(); + await writer; Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the write was logged"); Assert.That(seen, Is.EqualTo(new[] { One.Value }), "and published"); From 784dc7d6140f351dd379b0d4811130acfe69f1af Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 07:28:30 +0200 Subject: [PATCH 25/26] CultMesh: the untyped same-schema fallback reads through the database, which serves only what its shards own The string ResolveShard check was order-dependent in the full Mesh suite (schema id of a same-schema alias pair); GetAsync uses the descriptor path and adds no helper. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Mesh/CultMesh.cs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index bdc1dbdc..ef41d1f7 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -3370,20 +3370,6 @@ namespace GameCult.Mesh $"CultMesh document '{key.Value}' was not found as {typeof(TDocument).FullName}."); } - // A database serves only what its shards own; the cache behind it may hold more. - private static bool DatabaseServes(CultNetDatabase database, object document, CultRecordKey key) - { - try - { - database.ResolveShard(CultDocumentRegistry.Shared.GetRequired(document.GetType()).SchemaId, key); - return true; - } - catch (CultNetUnownedSchemaException) - { - return false; - } - } - private static async Task ReadDatabaseDocumentRequiredAsync( CultNetDatabase database, CultRecordKey key) @@ -3393,8 +3379,9 @@ namespace GameCult.Mesh if (document != null) return document; - var untyped = database.Cache.Get(key); - if (untyped != null && IsSameCultDocumentSchema(untyped.GetType()) && DatabaseServes(database, untyped, key)) + // Through the database, which serves only what its shards own; the cache behind it may hold more. + var untyped = await database.GetAsync(key).ConfigureAwait(false); + if (untyped != null && IsSameCultDocumentSchema(untyped.GetType())) return ConvertUntypedDocument(untyped); throw new KeyNotFoundException( From 3f942a57e777da3458a5d7c707ebcfe88a02f74a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 08:03:09 +0200 Subject: [PATCH 26/26] Cut 4 Soul 4d: a skipped entry is relayed as the primary sent it; probes, wire cases and the closed-enum rule committed F1: RecordUnappliedEntry keeps the primary's wire entry on the in-memory log entry (internal Relayed) and ToLogEntryMessage returns it through the same normalisation the store uses, so a replica with no store no longer relays a skipped put as a delete; memory and disk relay identically. Soul's 4d probes are committed (chained replica keeps the row, key-prefixed reads serve only owned keys, primary-all serves everything; the two measurements are explicit) with a memory/disk relay-parity test. Rust: captured-bytes cases for unowned_schema and variant_unsupported (with and without a subscription). The error-code set is stated as a closed enum in the schema description (both copies) and in cultnet-distributed-database.md: adding a code is one coordinated change across runtimes, since Rust and TypeScript refuse an unknown code. The Soul 4c probe file's header says what it guards. Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet/cultnet-distributed-database.md | 10 + contracts/cultnet/cultnet.error.schema.json | 2 +- .../cultnet/cultnet.error.schema.json | 2 +- packages/cultnet-rs/tests/error_message.rs | 40 ++++ src/GameCult.Networking/CultNetDatabase.cs | 11 +- .../SoulP4cProbeTests.cs | 4 +- .../SoulP4dProbeTests.cs | 201 ++++++++++++++++++ 7 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 8d2172c0..66bb450e 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -104,6 +104,16 @@ Not implemented yet: - cross-runtime rollback/resimulation helpers for simulation frames - declared CRDT merge policies +## Error Codes + +`cultnet.error.v0` carries an optional machine-readable `code`. The set of codes is a closed enum +(`cultnet.error.schema.json`): Rust and TypeScript refuse a message whose code they do not know, they do +not guess. Adding a code is therefore a coordinated change across every runtime, made together: the schema (both +copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in +`packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are +`selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported` and +`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns). + ## Live Invariants - CultCache owns document schema, identity, local indexes, and local diffing. diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index 0d541262..59764338 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -26,7 +26,7 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index 0d541262..59764338 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -26,7 +26,7 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 011ff147..5c633087 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -118,6 +118,46 @@ fn cases() -> Vec { details: None, }, }, + Case { + name: "variant_unsupported_subscription_ended", + // C#: ForVariantUnsupported("variant unsupported here", "sub-1") + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: Some(CultNetErrorDetails { + field: Some("subscriptionId".to_string()), + value: Some("sub-1".to_string()), + as_of: None, + current: None, + }), + }, + }, + Case { + name: "variant_unsupported_no_subscription", + // C#: ForVariantUnsupported("variant unsupported here") - no subscription ended, so Details stays null. + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: None, + }, + }, + Case { + name: "unowned_schema", + // C#: ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))) + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.".to_string(), + code: Some(CultNetErrorCode::UnownedSchema), + details: Some(CultNetErrorDetails { + field: Some("schemaId".to_string()), + value: Some("some.schema.v1".to_string()), + as_of: None, + current: None, + }), + }, + }, ] } diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index bedf91c6..668c088e 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -437,6 +437,10 @@ namespace GameCult.Networking /// Gets the previous document, when present. /// public object? PreviousDocument { get; } + + // The primary's own wire entry, kept for an entry this replica changed nothing for (a row its shard does not own, an + // absent key): it is relayed downstream exactly as the primary sent it, in memory and on disk alike. + internal CultNetShardLogEntryMessage? Relayed { get; set; } } /// @@ -1819,13 +1823,18 @@ namespace GameCult.Networking { RecordMutationLogEntry(new CultNetShardMutationLogEntry( shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, kind, - schemaId, key, document: null, previousDocument: null), entry); + schemaId, key, document: null, previousDocument: null) { Relayed = entry }, entry); _lastWriteSequence[(schemaId, key.Value)] = entry.Sequence; } } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) { + if (entry.Relayed != null) + { + return NormalizeWireLogEntry(entry, entry.Relayed); + } + if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null) { return new CultNetShardLogEntryMessage diff --git a/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs index 98956d54..a4796e7d 100644 --- a/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs +++ b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs @@ -11,7 +11,9 @@ using static GameCult.Networking.Tests.NetworkingTests; namespace GameCult.Networking.Tests { - // Soul pass 4c scratch probes (not for merge). + // Soul pass 4c probes. They guard: a committed put whose shard log failed is not answered as a refusal (Q1); reads and watch agree + // on what a database serves (Q2); a replica log apply leaves out a row its shard does not own (Q3); the unowned refusal is not a + // shard-authority exception (Q4); two databases over one cache aggregate their log failures (Q5). public sealed class SoulP4cProbeTests { private const string ShardId = "soulc-shard"; diff --git a/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs new file mode 100644 index 00000000..6623f8a1 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs @@ -0,0 +1,201 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4d probes. They guard the replica log relay (a skipped entry is relayed as the primary sent it, in memory and on + // disk), key-prefixed shard reads, and measure two costs; the measurements are explicit. + public sealed class SoulP4dProbeTests + { + private const string ShardId = "soupd-shard"; + private static readonly string Now = DateTimeOffset.UtcNow.ToString("O"); + + private static string NoteSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + private static string MeshSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, string[] schemas, string? keyPrefix = null, ICultNetShardMutationLogStore? store = null) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: schemas, keyPrefix: keyPrefix)], + MutationLogStore = store + }); + + private static CultNetShardLogResponseMessage UnownedPutLog(CultNetDatabase anyDb, string key) + { + var put = anyDb.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey(key)), new MeshQuickstartNote { NoteId = "p", Body = "primary-put" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + return new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = Now, ChangeKind = "added", Put = put }] + }; + } + + // Q3a: a replica (no store) that skips an unowned put relays it to a chained replica as a delete. + [Test, Category("SoulFinding")] + public async Task Q3a_MemoryLogRelaysASkippedPutAsARemoval() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)]); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + TestContext.Out.WriteLine($"memory relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null} delete={relayed[0].Delete != null}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"), "the relayed entry keeps the primary's kind"); + } + + // Q3b: the same replica with a durable store relays the primary's put (disk and memory disagree). + [Test, Category("SoulFinding")] + public async Task Q3b_StoreBackedLogRelaysTheSkippedPutAsAPut() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-" + Guid.NewGuid().ToString("N")); + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)], store: new CultNetFileShardMutationLogStore(dir)); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + var memory = replica.GetMutationLog(ShardId); + TestContext.Out.WriteLine($"store relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null}; memory entry kind={memory[0].Kind} doc={(memory[0].Document == null ? "null" : "set")}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added")); + } + + // Q3c: a chained replica that does own the row (config drift) and holds it deletes it on the relayed entry. + [Test, Category("SoulFinding")] + public async Task Q3c_AChainedReplicaThatOwnsTheRowDeletesWhatThePrimaryPut() + { + var cacheA = new CultCache(); + var a = Db(cacheA, primary: false, [NoteSchema(cacheA)]); + await a.ApplyShardLogResponseAsync(UnownedPutLog(a, "relay:k")); + + var cacheC = new CultCache(); + var c = Db(cacheC, primary: false, [NoteSchema(cacheC), MeshSchema(cacheC)]); + await cacheC.UpsertAsync(new MeshQuickstartNote { NoteId = "p", Body = "earlier" }, new CultRecordHandle(new CultRecordKey("relay:k"))); + await c.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "chain", + ShardId = ShardId, + ShardEpoch = 1, + Entries = a.GetMutationLogMessages(ShardId).ToArray() + }); + var row = cacheC.Get(new CultRecordKey("relay:k")); + TestContext.Out.WriteLine($"chained row after relay: {row?.Body ?? ""}"); + Assert.That(row, Is.Not.Null, "the primary put this row; a relay must not delete it"); + } + + // A skipped entry is relayed exactly as the primary sent it, whether the replica keeps its log in memory or on disk. + [Test] + public async Task SkippedEntriesAreRelayedIdenticallyFromMemoryAndFromDisk() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-parity-" + Guid.NewGuid().ToString("N")); + try + { + var memoryCache = new CultCache(); + var inMemory = Db(memoryCache, primary: false, [NoteSchema(memoryCache)]); + var diskCache = new CultCache(); + var onDisk = Db(diskCache, primary: false, [NoteSchema(diskCache)], store: new CultNetFileShardMutationLogStore(dir)); + var log = UnownedPutLog(inMemory, "relay:k"); + var removal = new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = Now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = memoryCache.Registry.GetRequired().SchemaId, + RecordKey = "relay:k", + ShardId = ShardId, + ShardEpoch = 1 + } + }; + log.Entries = [log.Entries[0], removal]; + + await inMemory.ApplyShardLogResponseAsync(log); + await onDisk.ApplyShardLogResponseAsync(log); + + var memory = inMemory.GetMutationLogMessages(ShardId); + var disk = onDisk.GetMutationLogMessages(ShardId); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(disk.Select(e => e.ChangeKind))); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(new[] { true, false })); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(disk.Select(e => e.Put != null))); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(new[] { false, true })); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(disk.Select(e => e.Delete != null))); + Assert.That(memory[0].Put!.Document!.RecordKey, Is.EqualTo(disk[0].Put!.Document!.RecordKey)); + Assert.That(memory[0].Put!.Document!.Payload, Is.EqualTo(disk[0].Put!.Document!.Payload), "the document bytes are the primary's"); + } + finally + { + if (Directory.Exists(dir)) Directory.Delete(dir, true); + } + } + + // Q2: key-prefixed shard: the schema is owned, a key outside the prefix is not. + [Test, Category("SoulPin")] + public async Task Q2_KeyPrefixedReadsServeOnlyTheOwnedKeys() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, [NoteSchema(cache)], keyPrefix: "a:"); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "a" }, new CultRecordHandle(new CultRecordKey("a:1"))); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "b" }, new CultRecordHandle(new CultRecordKey("b:1"))); + Assert.That(database.GetAll().Select(n => n.Text), Is.EqualTo(new[] { "a" })); + Assert.That(await database.GetAsync(new CultRecordKey("b:1")), Is.Null); + Assert.That(await database.GetAsync(new CultRecordKey("a:1")), Is.Not.Null); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var snapshot = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + Assert.That(snapshot.Documents.Select(d => d.RecordKey), Is.EqualTo(new[] { "a:1" })); + } + + [Test, Category("SoulPin")] + public async Task Q2_PrimaryAllServesEveryCachedRow() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { Shards = [CultNetShardDescriptor.PrimaryAll()] }); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "x", Body = "x" }, new CultRecordHandle(new CultRecordKey("any:1"))); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(database.GetByName("x"), Is.Not.Null); + } + + // Q2 cost: GetAll through the database versus the cache, scoped shard (reflection per row). + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion.")] + public void Q2_GetAllCost() + { + var cache = new CultCache(); + var database = Db(cache, primary: false, [NoteSchema(cache)], keyPrefix: "n:"); + cache.Commit(batch => + { + for (var i = 0; i < 20000; i++) + batch.Upsert(new NetworkSchemaNote { Text = "t" + i }, new CultRecordHandle(new CultRecordKey("n:" + i))); + }); + _ = cache.GetAll().Count(); + _ = database.GetAll().Count(); + var sw = Stopwatch.StartNew(); + for (var r = 0; r < 5; r++) _ = cache.GetAll().Count(); + var cacheMs = sw.Elapsed.TotalMilliseconds / 5; + sw.Restart(); + for (var r = 0; r < 5; r++) _ = database.GetAll().Count(); + var dbMs = sw.Elapsed.TotalMilliseconds / 5; + TestContext.Out.WriteLine($"GetAll 20000 rows: cache={cacheMs:F2}ms database={dbMs:F2}ms ratio={dbMs / Math.Max(cacheMs, 0.001):F1}x"); + } + + // Wire: the bytes a peer receives for the unowned refusal. + [Test, Category("SoulMeasure"), Explicit("Prints the wire bytes the Rust captured-bytes case pins.")] + public void Wire_UnownedSchemaBytes() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + var bytes = MessagePack.MessagePackSerializer.Serialize(message, CultNetSchemaMessageSerialization.Options); + TestContext.Out.WriteLine("HEX=" + Convert.ToHexString(bytes)); + TestContext.Out.WriteLine("JSON=" + MessagePack.MessagePackSerializer.ConvertToJson(bytes)); + } + } +}