diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index a70814d5..931088eb 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -47,10 +47,15 @@ which route wrote it. - Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a per-cache, in-memory number assigned when the cache admits the change, under its gate. It increases in admission order and is not persisted. - `OnUpdate(previous, current)` carries none. Streams CultNet derives from - `OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server, - database-backed Mesh handles) have no ordering guarantee and no stale - protection. + `OnUpdate(previous, current)` carries none. CultNet derives its + streams (`CultNetDatabase.WatchAllChanges`, the subscription server, + database-backed Mesh handles) from `Watch`, so a load, a commit and a variant + dependent's re-resolution all reach subscribers and the mutation log through + one handler over `Watch`; a database write door only hands that handler the + wire message or replicated entry for the change it admitted. That handler releases + changes in cache `Sequence` order, so the log and the stream agree with the + cache. A replica publishes but never mints log sequences. The + streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at or below it is reflected in the document. diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index d59cd6c7..359049f1 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using GameCult.Caching; @@ -472,7 +474,7 @@ namespace GameCult.Networking private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; - private readonly Dictionary _predictedDocuments = new(StringComparer.Ordinal); + private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); @@ -483,6 +485,26 @@ namespace GameCult.Networking private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); + private readonly IDisposable _cacheChanges; + // What a write door of this database knows about the change it is admitting and the cache does not: the wire + // message it carries, a prediction it reconciles, the replicated log entry it applies. The cache's Watch stream + // is the only publisher and the only logger; a door registers this context under the identity of the change it + // admits (the instance it writes, or the instance it removes) and the stream handler consumes it once, for that + // change alone. A write to the same key by anyone else is a different instance and gets no context. + private readonly ConcurrentDictionary _doors = new(ReferenceIdentity.Instance); + // One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a + // subscriber may write the cache from its handler. + private readonly object _logGate = new(); + // The cache admits a change to a key under its gate, but each thread reaches this handler after leaving it, in any + // order. Changes are therefore released strictly in cache Sequence order: a change ahead of a gap is parked until the + // gap fills, and one thread at a time drains the ready queue, outside every lock, so a subscriber that writes the + // cache from its handler queues behind the change it is handling instead of running inside it. + private readonly SortedDictionary _parked = new(); + private readonly Queue _ready = new(); + private long _nextSequence; + private bool _draining; + // Predictions this runtime made and has not seen reconciled. Every read and write takes _predictionGate. + private readonly object _predictionGate = new(); private bool _disposed; /// @@ -502,7 +524,12 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - _cache.OnUpdate += PublishCacheUpdate; + lock (_logGate) + { + _cacheChanges = _cache.Watch().Subscribe(OnCacheChange); + // Changes admitted before this read are released as they arrive; the next one is the first this handler waits for. + _nextSequence = _cache.GetWithSequence(new CultRecordKey(string.Empty)).Sequence + 1; + } } /// @@ -557,18 +584,21 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - if (!_mutationLogs.TryGetValue(shardId, out var entries)) + lock (_logGate) { - return Array.Empty(); - } + if (!_mutationLogs.TryGetValue(shardId, out var entries)) + { + return Array.Empty(); + } - var query = entries.Where(entry => entry.Sequence > afterSequence); - if (limit.HasValue) - { - query = query.Take(limit.Value); - } + var query = entries.Where(entry => entry.Sequence > afterSequence); + if (limit.HasValue) + { + query = query.Take(limit.Value); + } - return query.ToArray(); + return query.ToArray(); + } } /// @@ -700,39 +730,17 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - RemoveTrackedDocument(existing.DocumentType, existing.Key); - PublishUntyped( - existing.DocumentType, - CultNetDatabaseChangeKind.Removed, - existing.Key, - existing.SchemaId, - shard, - document: null, - previousDocument: existing.Document); + using (Door(existing.Document, new DoorContext(shard) { NoLog = true })) + _cache.Remove(existing.Key); } foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document))) { var key = new CultRecordKey(document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(document); - var previous = _cache.Get(key); - var applied = await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document, - ShardId = shard.ShardId, - ShardEpoch = shard.Epoch - }).ConfigureAwait(false); - PublishUntyped( - descriptor.DocumentType, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - applied, - previous); + var applied = _documents.DeserializeRawDocument(document); + using (Door(applied, new DoorContext(shard) { NoLog = true })) + await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false); } var sequence = response.ShardLogSequence ?? 0; @@ -846,27 +854,7 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - void PublishCommittedPut() - { - AppendMutationLogEntry( - shard, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - document, - previous); - Publish(new CultNetDatabaseChange( - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - document, - previous)); - } - PublishCommittedPut(); - return handle; + return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } /// @@ -881,16 +869,15 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsureClientAuthority(descriptor, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Predicted, - key, - descriptor.SchemaId, - shard, - document, - previous)); + CultRecordHandle handle; + using (Door(document, new DoorContext(shard) + { + Kind = CultNetDatabaseChangeKind.Predicted, + NoLog = true, + PredictionKey = PredictionKey(descriptor.SchemaId, key), + Predicts = true + })) + handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); return handle; } @@ -904,30 +891,7 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key); - var previous = _cache.Get(key); - if (previous != null) - { - _cache.Remove(new CultRecordHandle(key)); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous); - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous)); - } - PublishCommittedDelete(); - } - + _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; } @@ -947,42 +911,16 @@ namespace GameCult.Networking var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); - var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; + var document = _documents.DeserializeRawDocument(message.Document); var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); - if (reconcilesPrediction) + var reconcilesPrediction = HasPrediction(predictionKey); + using (Door(document, new DoorContext(shard) { - kind = CultNetDatabaseChangeKind.Reconciled; - } - - void PublishCommittedPut() - { - if (reconcilesPrediction) - _predictedDocuments.Remove(predictionKey); - AppendMutationLogEntry( - shard, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - key, - descriptor.SchemaId, - document, - previous, - new CultNetShardLogEntryMessage - { - ChangeKind = previous == null ? "added" : "updated", - Put = message - }); - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); - } - PublishCommittedPut(); + Put = message, + Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null, + PredictionKey = reconcilesPrediction ? predictionKey : null + })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); return document; } @@ -1004,31 +942,8 @@ namespace GameCult.Networking return Task.CompletedTask; } - _cache.Remove(key); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous, - wireEntry: new CultNetShardLogEntryMessage - { - ChangeKind = "removed", - Delete = message - }); - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); - } - PublishCommittedDelete(); + using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true })) + _cache.Remove(key); return Task.CompletedTask; } @@ -1154,40 +1069,234 @@ namespace GameCult.Networking } _disposed = true; - _cache.OnUpdate -= PublishCacheUpdate; + _cacheChanges.Dispose(); _changes.Dispose(); } - private void PublishCacheUpdate(object? previous, object? current) + // The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted + // change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in cache + // Sequence order. A replica publishes what it admits but never mints log sequences: its log holds only the primary's + // own entries, which the replica door hands over. + private void OnCacheChange(CultCacheDocumentChange change) { - var document = current ?? previous; - if (document == null) + var pending = new PendingChange(change, TakeDoor(change)); + lock (_logGate) + { + if (_disposed) + { + return; + } + + if (change.Sequence < _nextSequence) + { + _ready.Enqueue(pending); + } + else + { + _parked[change.Sequence] = pending; + while (_parked.Remove(_nextSequence, out var next)) + { + _ready.Enqueue(next); + _nextSequence++; + } + } + + if (_draining) + { + return; + } + + _draining = true; + } + + try + { + while (true) + { + PendingChange? item; + lock (_logGate) + { + if (_disposed || !_ready.TryDequeue(out item)) + { + _draining = false; + return; + } + } + + Deliver(item); + } + } + catch + { + lock (_logGate) + _draining = false; + throw; + } + } + + private void Deliver(PendingChange item) + { + var change = item.Change; + var door = item.Door; + var identity = change.Document ?? change.PreviousDocument; + if (identity == null) { return; } - var documentType = document.GetType(); + var documentType = identity.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); - var key = current != null - ? GetTrackedKey(current, documentType) - : new CultRecordKey(string.Empty); - var shard = ResolveShardInternal(descriptor, key); - var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); - var change = Activator.CreateInstance( - changeType, - current == null ? CultNetDatabaseChangeKind.Removed : - previous == null ? CultNetDatabaseChangeKind.Added : - CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - current, - previous, - null); - if (change != null) + var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key); + var kind = door?.Kind ?? change.Kind switch { - _changes.OnNext(change); + CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, + CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, + _ => CultNetDatabaseChangeKind.Updated + }; + var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + if (door?.PredictionKey is { } predictionKey) + { + lock (_predictionGate) + { + if (door.Predicts) + _predictedDocuments.Add(predictionKey); + else + _predictedDocuments.Remove(predictionKey); + } } + + if (door?.Replica is { } replicated) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + replicated.Sequence, + replicated.CommittedAt, + logKind, + descriptor.SchemaId, + change.Key, + change.Document, + change.PreviousDocument), + replicated); + _lastWriteSequence[(descriptor.SchemaId, change.Key.Value)] = replicated.Sequence; + } + } + else if (door?.NoLog != true && shard.IsPrimary) + { + AppendMutationLogEntry( + shard, + logKind, + change.Key, + descriptor.SchemaId, + change.Document, + change.PreviousDocument, + door?.Put != null + ? new CultNetShardLogEntryMessage + { + ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated", + Put = door.Put + } + : door?.Delete != null + ? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete } + : null); + } + + PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); + } + + private bool HasPrediction(string predictionKey) + { + lock (_predictionGate) + return _predictedDocuments.Contains(predictionKey); + } + + // A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had + // held, so its change carries that instance as the new document and never as the previous one; a removal door + // removes an instance, so its change carries it as the previous document and no new one. A plain removal of an + // instance a put door admitted, or an observer re-upserting that instance in place, is neither, and gets no context. + private DoorContext? TakeDoor(CultCacheDocumentChange change) + { + var identity = change.Document ?? change.PreviousDocument; + if (identity == null || !_doors.TryGetValue(identity, out var door)) + { + return null; + } + + var matches = change.Document == null + ? door.Removal + : !door.Removal && !ReferenceEquals(change.PreviousDocument, change.Document); + return matches && ((ICollection>)_doors) + .Remove(new KeyValuePair(identity, door)) + ? door + : null; + } + + private DoorRegistration Door(object identity, DoorContext context) => + new(_doors, identity, context, _doors.TryAdd(identity, context)); + + // Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind. + private readonly struct DoorRegistration : IDisposable + { + private readonly ConcurrentDictionary _doors; + private readonly object _identity; + private readonly DoorContext _context; + private readonly bool _owned; + + public DoorRegistration(ConcurrentDictionary doors, object identity, DoorContext context, bool owned) + { + _doors = doors; + _identity = identity; + _context = context; + _owned = owned; + } + + // True when the stream handler took the context: the door's write was admitted. + public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context); + + public void Dispose() + { + if (_owned) + ((ICollection>)_doors).Remove(new KeyValuePair(_identity, _context)); + } + } + + private sealed class PendingChange + { + public PendingChange(CultCacheDocumentChange change, DoorContext? door) + { + Change = change; + Door = door; + } + + public CultCacheDocumentChange Change { get; } + public DoorContext? Door { get; } + } + + private sealed class DoorContext + { + public DoorContext(CultNetShardDescriptor shard) => Shard = shard; + + public CultNetShardDescriptor Shard { get; } + public CultNetDatabaseChangeKind? Kind { get; set; } + public bool NoLog { get; set; } + public bool Removal { get; set; } + public CultNetDocumentPutRawMessage? Put { get; set; } + public CultNetDocumentDeleteMessage? Delete { get; set; } + public CultNetShardLogEntryMessage? Replica { get; set; } + // The prediction this door's change either records (Predicts) or ends. + public string? PredictionKey { get; set; } + public bool Predicts { get; set; } + } + + private sealed class ReferenceIdentity : IEqualityComparer + { + public static readonly ReferenceIdentity Instance = new(); + + bool IEqualityComparer.Equals(object? x, object? y) => ReferenceEquals(x, y); + + int IEqualityComparer.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value); } private CultRecordKey GetTrackedKey(object document, Type documentType) @@ -1240,16 +1349,6 @@ namespace GameCult.Networking } } - private void RemoveTrackedDocument(Type documentType, CultRecordKey key) - { - _cache.Remove(key); - } - - private void Publish(CultNetDatabaseChange change) where T : class - { - _changes.OnNext(change); - } - private sealed class TrackedShardDocument { public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document) @@ -1275,6 +1374,8 @@ namespace GameCult.Networking object? previousDocument, CultNetShardLogEntryMessage? wireEntry = null) { + lock (_logGate) + { var sequence = NextMutationLogSequence(shard.ShardId); var committedAt = DateTimeOffset.UtcNow.ToString("O"); var entry = new CultNetShardMutationLogEntry( @@ -1292,6 +1393,7 @@ namespace GameCult.Networking : wireEntry ?? ToLogEntryMessage(entry); RecordMutationLogEntry(entry, storedWireEntry); _lastWriteSequence[(schemaId, key.Value)] = sequence; + } } /// @@ -1299,8 +1401,11 @@ namespace GameCult.Networking /// never been committed through this database. The order the CultNet selection evaluator /// pages by (docs/cultnet-selection-cut.md, section 2). /// - public long? LastWriteSequence(string schemaId, CultRecordKey key) => - _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + public long? LastWriteSequence(string schemaId, CultRecordKey key) + { + lock (_logGate) + return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + } /// /// The current write-sequence watermark across every shard this database holds, 0 when nothing @@ -1311,7 +1416,11 @@ namespace GameCult.Networking /// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to /// while the page's own shard sat still. /// - public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + public ulong CurrentAsOf() + { + lock (_logGate) + return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + } /// /// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit @@ -1325,7 +1434,8 @@ namespace GameCult.Networking public ulong CurrentAsOf(string shardId) { if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; + lock (_logGate) + return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; } private void RecordMutationLogEntry( @@ -1453,8 +1563,6 @@ namespace GameCult.Networking var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { @@ -1463,35 +1571,17 @@ namespace GameCult.Networking } var predictionKey = PredictionKey(descriptor.SchemaId, key); - if (_predictedDocuments.Remove(predictionKey)) + var reconcilesPrediction = HasPrediction(predictionKey); + if (reconcilesPrediction) { kind = CultNetDatabaseChangeKind.Reconciled; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - descriptor.SchemaId, - key, - document, - previous), - entry); - // R-Q: a replica apply is a commit path too - the last-write sequence this row now carries - // is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets - // it, so the selection evaluator's order and cursor agree whether a row's most recent write - // landed locally or arrived through replication. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); + // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that + // admits the change; the stream handler does both. + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = reconcilesPrediction ? predictionKey : null })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } private void ApplyCommittedDelete( @@ -1510,33 +1600,27 @@ namespace GameCult.Networking var key = new CultRecordKey(message.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); + var admitted = false; var previous = _cache.Get(key); if (previous != null) { + using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true }); _cache.Remove(key); + admitted = door.Consumed; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - CultNetDatabaseChangeKind.Removed, - descriptor.SchemaId, - key, - document: null, - previousDocument: previous), - entry); - // R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); + if (!admitted) + { + // The key is not here, so nothing was admitted to log. This replica's log must still hold every entry of the + // primary's, or a replica chained on it sees a gap. + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed, + descriptor.SchemaId, key, document: null, previousDocument: null), entry); + _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; + } + } } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) diff --git a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs new file mode 100644 index 00000000..65e290a4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs @@ -0,0 +1,347 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + public sealed class CultNetPublicationOrderTests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text; + + // A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed + // (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile. + [Test] + public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "A") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache); + var seen = new List(); + using var subscriber = database.Watch().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); }); + var key = new CultRecordKey("soul:order"); + + var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle(key)).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True); + await database.PutAsync(key, Note("B")); + release.Set(); + await a; + + var shard = database.Shards[0].ShardId; + var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last(); + + // A replica built from the log. + var replicaCache = new CultCache(); + var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)] + }); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shard, + ShardEpoch = database.Shards[0].Epoch, + Entries = database.GetMutationLogMessages(shard).ToArray() + }); + + Assert.Multiple(() => + { + Assert.That(cache.Get(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B"); + Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write"); + Assert.That(replicaCache.Get(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary"); + Assert.That(seen.Last(), Is.EqualTo("B"), "a subscriber's final state for the key"); + }); + } + + // Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache. + [Test] + public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing() + { + var divergent = 0; + const int rounds = 300; + for (var round = 0; round < rounds; round++) + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:hammer"); + using var start = new Barrier(2); + Parallel.For(0, 2, writer => + { + start.SignalAndWait(); + for (var i = 0; i < 40; i++) + cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle(key)).GetAwaiter().GetResult(); + }); + var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last(); + if (!ReferenceEquals(last.Document, cache.Get(key))) + divergent++; + } + + TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}"); + Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache"); + } + + // A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in + // the window before the door's own change is handled, consumes the door's context. + [Test] + public async Task S2_ARemovalNeverWearsAReconcilingPutsContext() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:reconcile"); + await database.PutPredictedAsync(key, Note("predicted")); + var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>(); + using var subscriber = database.Watch().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text))); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + var a = Task.Run(() => database.ApplyPutAsync(message)); + Assert.That(entered.Wait(Wait), Is.True); + cache.Remove(key); + release.Set(); + await a; + + var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray(); + Assert.Multiple(() => + { + Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1), + "the removal is published as a removal: " + string.Join(", ", published)); + Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text), + Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document"); + Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True, + "no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document)))); + }); + } + + // A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not + // hold (B removed it locally, a supported write) must still keep B's log contiguous for C. + [Test] + public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey() + { + const string shardId = "chain"; + var schemaId = new CultCache().Registry.GetRequired().SchemaId; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])] + }); + var p = Make(new CultCache(), true); + var bCache = new CultCache(); + var b = Make(bCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray() + }); + } + + var k = new CultRecordKey("soul:chain:k"); + await p.PutAsync(k, Note("x")); + await Pull(p, b); + await Pull(b, c); + bCache.Remove(k); + await p.DeleteAsync(k); + await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y")); + await Pull(p, b); + + Assert.Multiple(() => + { + Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }), + "B's log holds every primary entry it applied"); + Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap"); + }); + } + + // A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction. + [Test] + public async Task S4_APredictionMadeInReactionToAReconcileSurvives() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:repredict"); + var kinds = new ConcurrentQueue(); + var repredicted = false; + using var subscriber = database.Watch().Subscribe(change => + { + kinds.Enqueue(change.Kind); + if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted) + { + repredicted = true; + database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult(); + } + }); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled + })); + } + // Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A + // refused prediction leaves nothing behind that a later write of the same instance would wear. + [Test] + public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var baseKey = new CultRecordKey("soul-refuse-base"); + var variantKey = new CultRecordKey("soul-refuse-variant"); + await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 }, + new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v") + }); + var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 }; + Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException); + var kinds = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => kinds.Enqueue(change.Kind)); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await cache.UpsertAsync(instance, new CultRecordHandle(new CultRecordKey("soul-refuse-other"))); + + Assert.Multiple(() => + { + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1)); + }); + } + + // Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a + // durable log store can observe even though the disposed Subject hides publication. + [Test] + public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog() + { + var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N")); + var store = new CultNetFileShardMutationLogStore(root); + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store }); + var shard = database.Shards[0].ShardId; + await cache.UpsertAsync(Note("before"), new CultRecordHandle(new CultRecordKey("soul:dispose:before"))); + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + database.Dispose(); + + try { await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("soul:dispose:after"))); } + catch { /* a throwing handler would also be a failure below */ } + + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + System.IO.Directory.Delete(root, true); + } + + // The log entry exists before its change is published: a subscriber that reads the log from its handler finds it. + [Test] + public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:log-first"); + var loggedWhenPublished = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => + loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value))); + + await database.PutAsync(key, Note("one")); + cache.Remove(key); + + Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true })); + } + + // An observer that mutates the admitted instance in place and upserts it again writes a second change of the same + // instance. It is published as its own update and never wears the context of the door's change, even when it + // reaches the database first. + [Test] + public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext() + { + var cache = new CultCache(); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + change.Document.Text = "normalized"; + cache.UpsertAsync(change.Document, new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:in-place"); + var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>(); + using var subscriber = database.Watch().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text))); + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(message); + + Assert.That(seen, Is.EqualTo(new (CultNetDatabaseChangeKind, string?)[] + { + (CultNetDatabaseChangeKind.Predicted, null), + (CultNetDatabaseChangeKind.Reconciled, "predicted"), + (CultNetDatabaseChangeKind.Updated, "normalized") + })); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs new file mode 100644 index 00000000..fbd1df2a --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs @@ -0,0 +1,429 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; +using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests; + +namespace GameCult.Networking.Tests +{ + // The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the + // mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every + // other change in the hold publishes from the cache. + public sealed class CultNetPublishesCommitsTests + { + private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) => + wire.Changes.Where(change => KeyOf(change) == key).ToArray(); + + private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey; + + private sealed class Wire : IDisposable + { + private readonly CancellationTokenSource _cancellation = new(); + private readonly Thread _thread; + private readonly RudpCultNetSchemaServer _server; + private readonly CultNetDatabaseSubscriptionServer _subscriptions; + private readonly ICultNetSchemaClient _client; + public ConcurrentQueue Changes { get; } = new(); + public int Snapshots; + public ConcurrentQueue Errors { get; } = new(); + + private readonly CultNetDatabase _database; + + public Wire(CultNetDatabase database) + { + _database = database; + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + _server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket }); + _subscriptions = new CultNetDatabaseSubscriptionServer(_server, database); + _thread = new Thread(() => + { + while (!_cancellation.IsCancellationRequested) + { + _ = _server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + _thread.Start(); + _client = CultNetSchemaClients.CreateRudp("publish-client"); + _client.OnCultNet(Changes.Enqueue); + _client.OnCultNet(Errors.Enqueue); + _client.OnCultNet(_ => Interlocked.Increment(ref Snapshots)); + _client.Connect("127.0.0.1", _server.LocalEndPoint.Port); + } + + public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes) + { + await Until(() => _client.Connected); + _client.SendCultNet(new CultNetDatabaseSubscribeMessage + { + MessageId = "subscribe", + SubscriptionId = "all", + SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(), + IncludeSnapshot = includeSnapshot + }); + await Until(() => Snapshots > 0 || !Errors.IsEmpty); + Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused"); + } + + public void Dispose() + { + _cancellation.Cancel(); + _thread.Join(); + _client.Dispose(); + _subscriptions.Dispose(); + _server.Dispose(); + } + } + + private static async Task Until(Func condition) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + Assert.Fail("Condition was not satisfied before timeout."); + await Task.Delay(5); + } + } + + // One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance + // to arrive twice. + private static async Task SettleAsync(CultNetDatabase database, Wire wire) + { + var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N"); + await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" }); + await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel)); + } + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + [Test] + public async Task ACacheCommitAndUpsertReachASubscriberOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:commit"); + + cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle(key))); + await Until(() => Of(wire, key.Value).Length >= 1); + cache.Commit(batch => batch.Remove(key)); + await Until(() => Of(wire, key.Value).Length >= 2); + await cache.UpsertAsync(Note("upserted"), new CultRecordHandle(key)); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3)); + } + + [Test] + public async Task ADatabasePutAndDeleteStillPublishExactlyOnce() + { + var database = new CultNetDatabase(new CultCache()); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:put"); + + await database.PutAsync(key, Note("put")); + await database.DeleteAsync(key); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + [Test] + public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:replicated"); + var message = database.Documents.CreateRawDocumentPutMessage( + "put-replicated", new CultRecordHandle(key), Note("remote")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.ApplyPutAsync(message); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1)); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1)); + } + + [Test] + public async Task ALoadRemovalCarriesItsKey() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc"); + try + { + var key = new CultRecordKey("tests:publish:loaded"); + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("there"), new CultRecordHandle(key)); + writer.FlushAllBackingStores(); + + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var database = new CultNetDatabase(cache); + Assert.That(cache.Get(key), Is.Not.Null); + using var wire = new Wire(database); + await wire.SubscribeAsync(true, typeof(NetworkSchemaNote)); + + writer.Remove(key); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big"), + cache.Override(nameof(VariantWireFixture.Power), 9) + }); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(VariantWireFixture)); + var errorsBefore = wire.Errors.Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + await Until(() => wire.Errors.Count > errorsBefore); + await Until(() => Of(wire, baseKey.Value).Length >= 1); + + Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value), + "the re-resolved dependent is published, then refused (Q6) by key"); + Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1)); + Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record"); + } + + // An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own. + [Test] + public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:normalized"); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "raw") + cache.UpsertAsync(Note("normalized"), new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + }); + + await database.PutAsync(key, Note("raw")); + await Until(() => Of(wire, key.Value).Length >= 2); + await SettleAsync(database, wire); + + Assert.That(cache.Get(key)!.Text, Is.EqualTo("normalized")); + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + return (cache, documents); + } + + private static System.Collections.Generic.List Record(CultNetDatabase database, Func describe) + { + var seen = new System.Collections.Generic.List(); + database.WatchAllChanges().Subscribe(change => seen.Add(describe(change))); + return seen; + } + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + [Test] + public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder() + { + var (cache, documents) = VariantCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big") + }); + var published = Record(database, KeyOfChange); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value), + Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + } + + private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) => + new(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + Shards = + [ + new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId]) + ] + }); + + [Test] + public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally() + { + var (sourceCache, sourceDocuments) = VariantCache(); + var (replicaCache, replicaDocuments) = VariantCache(); + var schemaId = sourceCache.Registry.GetRequired().SchemaId; + var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions + { + DocumentRegistry = sourceDocuments, + Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])] + }); + var replica = Replica(replicaCache, replicaDocuments, schemaId); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[] + { + replicaCache.Override(nameof(VariantWireFixture.Name), "big") + }); + await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Entries = source.GetMutationLogMessages("replica-shard").ToArray() + }); + + Assert.That(replicaCache.Get(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved"); + Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }), + "the replica log holds the primary's entry and mints none of its own"); + } + + [Test] + public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var published = Record(replica, KeyOfChange); + + await cache.UpsertAsync(Note("local"), new CultRecordHandle(new CultRecordKey("tests:publish:local"))); + + Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + const int writers = 4, perWriter = 5000; + + Parallel.For(0, writers, writer => + { + for (var i = 0; i < perWriter; i++) + cache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey($"tests:publish:c:{writer}:{i}"))) + .GetAwaiter().GetResult(); + }); + + var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray(); + Assert.That(sequences, Has.Length.EqualTo(writers * perWriter)); + Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter)); + } + + [Test] + public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("tests:publish:after-delete"); + await database.PutAsync(key, Note("one")); + await database.DeleteAsync(key); + var published = Record(database, KeyOfChange); + + await cache.UpsertAsync(Note("two"), new CultRecordHandle(key)); + + Assert.That(published, Is.EqualTo(new[] { key.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind), + Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added + })); + } + + [Test] + public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var stale = new CultRecordKey("tests:publish:stale"); + var fresh = new CultRecordKey("tests:publish:fresh"); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(stale)); + var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value); + var put = replica.Documents.CreateRawDocumentPutMessage( + "snapshot-fresh", new CultRecordHandle(fresh), Note("fresh")); + + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public async Task ADisposedDatabaseNoLongerObservesTheCache() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var published = Record(database, KeyOfChange); + database.Dispose(); + + await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("tests:publish:disposed"))); + + Assert.That(published, Is.Empty); + } + } +} diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index e32ca91a..d9ee9704 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(batched), Is.Null); }), Is.True); - Assert.That(published, Is.EqualTo(new[] { put })); + Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it"); Assert.That(cache.Get(batched)?.Text, Is.EqualTo("batched")); var reopened = new CultCache(); reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); @@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests var cache = new CultCache(); var delivering = new ManualResetEventSlim(); - cache.OnUpdate += (_, _) => delivering.Set(); + // Subscribed before the database, so it runs before the database's own observer of the same change. + using var loadDelivering = cache.Watch().Subscribe(_ => delivering.Set()); var database = new CultNetDatabase(cache); var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions {