Merge the salvaged CultNet publication branch (queue removed in the next commit)
Cut 4a: brings the publication tests and the door table from
hands/cultnet-publish-commits at 28248ebe. The reorder buffer, drainer and
prediction gate die in 4b.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
7d945b002d
5 changed files with 1109 additions and 243 deletions
|
|
@ -47,10 +47,15 @@ which route wrote it.
|
|||
- Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a
|
||||
per-cache, in-memory number assigned when the cache admits the change, under
|
||||
its gate. It increases in admission order and is not persisted.
|
||||
`OnUpdate(previous, current)` carries none. Streams CultNet derives from
|
||||
`OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server,
|
||||
database-backed Mesh handles) have no ordering guarantee and no stale
|
||||
protection.
|
||||
`OnUpdate(previous, current)` carries none. CultNet derives its
|
||||
streams (`CultNetDatabase.WatchAllChanges`, the subscription server,
|
||||
database-backed Mesh handles) from `Watch`, so a load, a commit and a variant
|
||||
dependent's re-resolution all reach subscribers and the mutation log through
|
||||
one handler over `Watch`; a database write door only hands that handler the
|
||||
wire message or replicated entry for the change it admitted. That handler releases
|
||||
changes in cache `Sequence` order, so the log and the stream agree with the
|
||||
cache. A replica publishes but never mints log sequences. The
|
||||
streams carry no `Sequence` and have no stale protection.
|
||||
- `GetWithSequence(key)` returns the document and the cache's current
|
||||
`Sequence`, read together under the gate: every change with a `Sequence` at
|
||||
or below it is reflected in the document.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
|
|
@ -472,7 +474,7 @@ namespace GameCult.Networking
|
|||
private readonly List<CultNetShardDescriptor> _shards;
|
||||
private readonly List<CultNetClientAuthorityScope> _clientAuthorityScopes;
|
||||
private readonly ICultNetShardMutationLogStore? _mutationLogStore;
|
||||
private readonly Dictionary<string, object> _predictedDocuments = new(StringComparer.Ordinal);
|
||||
private readonly HashSet<string> _predictedDocuments = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<CultNetShardMutationLogEntry>> _mutationLogs =
|
||||
new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _nextLogSequences = new(StringComparer.Ordinal);
|
||||
|
|
@ -483,6 +485,26 @@ namespace GameCult.Networking
|
|||
private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence =
|
||||
new();
|
||||
private readonly Subject<object> _changes = new();
|
||||
private readonly IDisposable _cacheChanges;
|
||||
// What a write door of this database knows about the change it is admitting and the cache does not: the wire
|
||||
// message it carries, a prediction it reconciles, the replicated log entry it applies. The cache's Watch stream
|
||||
// is the only publisher and the only logger; a door registers this context under the identity of the change it
|
||||
// admits (the instance it writes, or the instance it removes) and the stream handler consumes it once, for that
|
||||
// change alone. A write to the same key by anyone else is a different instance and gets no context.
|
||||
private readonly ConcurrentDictionary<object, DoorContext> _doors = new(ReferenceIdentity.Instance);
|
||||
// One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a
|
||||
// subscriber may write the cache from its handler.
|
||||
private readonly object _logGate = new();
|
||||
// The cache admits a change to a key under its gate, but each thread reaches this handler after leaving it, in any
|
||||
// order. Changes are therefore released strictly in cache Sequence order: a change ahead of a gap is parked until the
|
||||
// gap fills, and one thread at a time drains the ready queue, outside every lock, so a subscriber that writes the
|
||||
// cache from its handler queues behind the change it is handling instead of running inside it.
|
||||
private readonly SortedDictionary<long, PendingChange> _parked = new();
|
||||
private readonly Queue<PendingChange> _ready = new();
|
||||
private long _nextSequence;
|
||||
private bool _draining;
|
||||
// Predictions this runtime made and has not seen reconciled. Every read and write takes _predictionGate.
|
||||
private readonly object _predictionGate = new();
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -502,7 +524,12 @@ namespace GameCult.Networking
|
|||
_clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty<CultNetClientAuthorityScope>()).ToList();
|
||||
CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random();
|
||||
InitializeLogSequencesFromStore();
|
||||
_cache.OnUpdate += PublishCacheUpdate;
|
||||
lock (_logGate)
|
||||
{
|
||||
_cacheChanges = _cache.Watch<object>().Subscribe(OnCacheChange);
|
||||
// Changes admitted before this read are released as they arrive; the next one is the first this handler waits for.
|
||||
_nextSequence = _cache.GetWithSequence(new CultRecordKey(string.Empty)).Sequence + 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -557,18 +584,21 @@ namespace GameCult.Networking
|
|||
{
|
||||
ThrowIfDisposed();
|
||||
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
|
||||
if (!_mutationLogs.TryGetValue(shardId, out var entries))
|
||||
lock (_logGate)
|
||||
{
|
||||
return Array.Empty<CultNetShardMutationLogEntry>();
|
||||
}
|
||||
if (!_mutationLogs.TryGetValue(shardId, out var entries))
|
||||
{
|
||||
return Array.Empty<CultNetShardMutationLogEntry>();
|
||||
}
|
||||
|
||||
var query = entries.Where(entry => entry.Sequence > afterSequence);
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
var query = entries.Where(entry => entry.Sequence > afterSequence);
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
return query.ToArray();
|
||||
return query.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -700,39 +730,17 @@ namespace GameCult.Networking
|
|||
var local = GetLocalShardDocuments(shard).ToArray();
|
||||
foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value)))
|
||||
{
|
||||
RemoveTrackedDocument(existing.DocumentType, existing.Key);
|
||||
PublishUntyped(
|
||||
existing.DocumentType,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
existing.Key,
|
||||
existing.SchemaId,
|
||||
shard,
|
||||
document: null,
|
||||
previousDocument: existing.Document);
|
||||
using (Door(existing.Document, new DoorContext(shard) { NoLog = true }))
|
||||
_cache.Remove(existing.Key);
|
||||
}
|
||||
|
||||
foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document)))
|
||||
{
|
||||
var key = new CultRecordKey(document.RecordKey);
|
||||
var descriptor = _documents.ResolveDescriptorForRawDocument(document);
|
||||
var previous = _cache.Get(key);
|
||||
var applied = await _documents.ApplyRawDocumentPutMessageAsync(
|
||||
_cache,
|
||||
new CultNetDocumentPutRawMessage
|
||||
{
|
||||
MessageId = response.MessageId,
|
||||
Document = document,
|
||||
ShardId = shard.ShardId,
|
||||
ShardEpoch = shard.Epoch
|
||||
}).ConfigureAwait(false);
|
||||
PublishUntyped(
|
||||
descriptor.DocumentType,
|
||||
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
applied,
|
||||
previous);
|
||||
var applied = _documents.DeserializeRawDocument(document);
|
||||
using (Door(applied, new DoorContext(shard) { NoLog = true }))
|
||||
await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var sequence = response.ShardLogSequence ?? 0;
|
||||
|
|
@ -846,27 +854,7 @@ namespace GameCult.Networking
|
|||
var shard = ResolveShardInternal(descriptor, key);
|
||||
EnsurePrimary(shard, descriptor.SchemaId, key);
|
||||
|
||||
var previous = _cache.Get<T>(key);
|
||||
var handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
|
||||
void PublishCommittedPut()
|
||||
{
|
||||
AppendMutationLogEntry(
|
||||
shard,
|
||||
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
document,
|
||||
previous);
|
||||
Publish(new CultNetDatabaseChange<T>(
|
||||
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document,
|
||||
previous));
|
||||
}
|
||||
PublishCommittedPut();
|
||||
return handle;
|
||||
return await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -881,16 +869,15 @@ namespace GameCult.Networking
|
|||
var shard = ResolveShardInternal(descriptor, key);
|
||||
EnsureClientAuthority(descriptor, key);
|
||||
|
||||
var previous = _cache.Get<T>(key);
|
||||
var handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
|
||||
_predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document;
|
||||
Publish(new CultNetDatabaseChange<T>(
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document,
|
||||
previous));
|
||||
CultRecordHandle<T> handle;
|
||||
using (Door(document, new DoorContext(shard)
|
||||
{
|
||||
Kind = CultNetDatabaseChangeKind.Predicted,
|
||||
NoLog = true,
|
||||
PredictionKey = PredictionKey(descriptor.SchemaId, key),
|
||||
Predicts = true
|
||||
}))
|
||||
handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
|
||||
return handle;
|
||||
}
|
||||
|
||||
|
|
@ -904,30 +891,7 @@ namespace GameCult.Networking
|
|||
var shard = ResolveShardInternal(descriptor, key);
|
||||
EnsurePrimary(shard, descriptor.SchemaId, key);
|
||||
|
||||
var previous = _cache.Get<T>(key);
|
||||
if (previous != null)
|
||||
{
|
||||
_cache.Remove(new CultRecordHandle<T>(key));
|
||||
void PublishCommittedDelete()
|
||||
{
|
||||
AppendMutationLogEntry(
|
||||
shard,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
document: null,
|
||||
previousDocument: previous);
|
||||
Publish(new CultNetDatabaseChange<T>(
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document: null,
|
||||
previousDocument: previous));
|
||||
}
|
||||
PublishCommittedDelete();
|
||||
}
|
||||
|
||||
_cache.Remove(new CultRecordHandle<T>(key));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
|
|
@ -947,42 +911,16 @@ namespace GameCult.Networking
|
|||
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
|
||||
var shard = ResolveShardInternal(descriptor, key);
|
||||
EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch);
|
||||
var previous = _cache.Get(key);
|
||||
var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false);
|
||||
var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated;
|
||||
var document = _documents.DeserializeRawDocument(message.Document);
|
||||
var predictionKey = PredictionKey(descriptor.SchemaId, key);
|
||||
var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey);
|
||||
if (reconcilesPrediction)
|
||||
var reconcilesPrediction = HasPrediction(predictionKey);
|
||||
using (Door(document, new DoorContext(shard)
|
||||
{
|
||||
kind = CultNetDatabaseChangeKind.Reconciled;
|
||||
}
|
||||
|
||||
void PublishCommittedPut()
|
||||
{
|
||||
if (reconcilesPrediction)
|
||||
_predictedDocuments.Remove(predictionKey);
|
||||
AppendMutationLogEntry(
|
||||
shard,
|
||||
kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
document,
|
||||
previous,
|
||||
new CultNetShardLogEntryMessage
|
||||
{
|
||||
ChangeKind = previous == null ? "added" : "updated",
|
||||
Put = message
|
||||
});
|
||||
PublishUntyped(
|
||||
descriptor.DocumentType,
|
||||
kind,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document,
|
||||
previous);
|
||||
}
|
||||
PublishCommittedPut();
|
||||
Put = message,
|
||||
Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null,
|
||||
PredictionKey = reconcilesPrediction ? predictionKey : null
|
||||
}))
|
||||
await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false);
|
||||
return document;
|
||||
}
|
||||
|
||||
|
|
@ -1004,31 +942,8 @@ namespace GameCult.Networking
|
|||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
_cache.Remove(key);
|
||||
void PublishCommittedDelete()
|
||||
{
|
||||
AppendMutationLogEntry(
|
||||
shard,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
document: null,
|
||||
previousDocument: previous,
|
||||
wireEntry: new CultNetShardLogEntryMessage
|
||||
{
|
||||
ChangeKind = "removed",
|
||||
Delete = message
|
||||
});
|
||||
PublishUntyped(
|
||||
descriptor.DocumentType,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document: null,
|
||||
previousDocument: previous);
|
||||
}
|
||||
PublishCommittedDelete();
|
||||
using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true }))
|
||||
_cache.Remove(key);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
|
|
@ -1154,40 +1069,234 @@ namespace GameCult.Networking
|
|||
}
|
||||
|
||||
_disposed = true;
|
||||
_cache.OnUpdate -= PublishCacheUpdate;
|
||||
_cacheChanges.Dispose();
|
||||
_changes.Dispose();
|
||||
}
|
||||
|
||||
private void PublishCacheUpdate(object? previous, object? current)
|
||||
// The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted
|
||||
// change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in cache
|
||||
// Sequence order. A replica publishes what it admits but never mints log sequences: its log holds only the primary's
|
||||
// own entries, which the replica door hands over.
|
||||
private void OnCacheChange(CultCacheDocumentChange<object> change)
|
||||
{
|
||||
var document = current ?? previous;
|
||||
if (document == null)
|
||||
var pending = new PendingChange(change, TakeDoor(change));
|
||||
lock (_logGate)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (change.Sequence < _nextSequence)
|
||||
{
|
||||
_ready.Enqueue(pending);
|
||||
}
|
||||
else
|
||||
{
|
||||
_parked[change.Sequence] = pending;
|
||||
while (_parked.Remove(_nextSequence, out var next))
|
||||
{
|
||||
_ready.Enqueue(next);
|
||||
_nextSequence++;
|
||||
}
|
||||
}
|
||||
|
||||
if (_draining)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_draining = true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
PendingChange? item;
|
||||
lock (_logGate)
|
||||
{
|
||||
if (_disposed || !_ready.TryDequeue(out item))
|
||||
{
|
||||
_draining = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Deliver(item);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
lock (_logGate)
|
||||
_draining = false;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void Deliver(PendingChange item)
|
||||
{
|
||||
var change = item.Change;
|
||||
var door = item.Door;
|
||||
var identity = change.Document ?? change.PreviousDocument;
|
||||
if (identity == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var documentType = document.GetType();
|
||||
var documentType = identity.GetType();
|
||||
var descriptor = _cache.Registry.GetRequired(documentType);
|
||||
var key = current != null
|
||||
? GetTrackedKey(current, documentType)
|
||||
: new CultRecordKey(string.Empty);
|
||||
var shard = ResolveShardInternal(descriptor, key);
|
||||
var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType);
|
||||
var change = Activator.CreateInstance(
|
||||
changeType,
|
||||
current == null ? CultNetDatabaseChangeKind.Removed :
|
||||
previous == null ? CultNetDatabaseChangeKind.Added :
|
||||
CultNetDatabaseChangeKind.Updated,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
current,
|
||||
previous,
|
||||
null);
|
||||
if (change != null)
|
||||
var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key);
|
||||
var kind = door?.Kind ?? change.Kind switch
|
||||
{
|
||||
_changes.OnNext(change);
|
||||
CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed,
|
||||
CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added,
|
||||
_ => CultNetDatabaseChangeKind.Updated
|
||||
};
|
||||
var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind;
|
||||
if (door?.PredictionKey is { } predictionKey)
|
||||
{
|
||||
lock (_predictionGate)
|
||||
{
|
||||
if (door.Predicts)
|
||||
_predictedDocuments.Add(predictionKey);
|
||||
else
|
||||
_predictedDocuments.Remove(predictionKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (door?.Replica is { } replicated)
|
||||
{
|
||||
lock (_logGate)
|
||||
{
|
||||
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
|
||||
shard.ShardId,
|
||||
shard.Epoch,
|
||||
replicated.Sequence,
|
||||
replicated.CommittedAt,
|
||||
logKind,
|
||||
descriptor.SchemaId,
|
||||
change.Key,
|
||||
change.Document,
|
||||
change.PreviousDocument),
|
||||
replicated);
|
||||
_lastWriteSequence[(descriptor.SchemaId, change.Key.Value)] = replicated.Sequence;
|
||||
}
|
||||
}
|
||||
else if (door?.NoLog != true && shard.IsPrimary)
|
||||
{
|
||||
AppendMutationLogEntry(
|
||||
shard,
|
||||
logKind,
|
||||
change.Key,
|
||||
descriptor.SchemaId,
|
||||
change.Document,
|
||||
change.PreviousDocument,
|
||||
door?.Put != null
|
||||
? new CultNetShardLogEntryMessage
|
||||
{
|
||||
ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated",
|
||||
Put = door.Put
|
||||
}
|
||||
: door?.Delete != null
|
||||
? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete }
|
||||
: null);
|
||||
}
|
||||
|
||||
PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument);
|
||||
}
|
||||
|
||||
private bool HasPrediction(string predictionKey)
|
||||
{
|
||||
lock (_predictionGate)
|
||||
return _predictedDocuments.Contains(predictionKey);
|
||||
}
|
||||
|
||||
// A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had
|
||||
// held, so its change carries that instance as the new document and never as the previous one; a removal door
|
||||
// removes an instance, so its change carries it as the previous document and no new one. A plain removal of an
|
||||
// instance a put door admitted, or an observer re-upserting that instance in place, is neither, and gets no context.
|
||||
private DoorContext? TakeDoor(CultCacheDocumentChange<object> change)
|
||||
{
|
||||
var identity = change.Document ?? change.PreviousDocument;
|
||||
if (identity == null || !_doors.TryGetValue(identity, out var door))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var matches = change.Document == null
|
||||
? door.Removal
|
||||
: !door.Removal && !ReferenceEquals(change.PreviousDocument, change.Document);
|
||||
return matches && ((ICollection<KeyValuePair<object, DoorContext>>)_doors)
|
||||
.Remove(new KeyValuePair<object, DoorContext>(identity, door))
|
||||
? door
|
||||
: null;
|
||||
}
|
||||
|
||||
private DoorRegistration Door(object identity, DoorContext context) =>
|
||||
new(_doors, identity, context, _doors.TryAdd(identity, context));
|
||||
|
||||
// Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind.
|
||||
private readonly struct DoorRegistration : IDisposable
|
||||
{
|
||||
private readonly ConcurrentDictionary<object, DoorContext> _doors;
|
||||
private readonly object _identity;
|
||||
private readonly DoorContext _context;
|
||||
private readonly bool _owned;
|
||||
|
||||
public DoorRegistration(ConcurrentDictionary<object, DoorContext> doors, object identity, DoorContext context, bool owned)
|
||||
{
|
||||
_doors = doors;
|
||||
_identity = identity;
|
||||
_context = context;
|
||||
_owned = owned;
|
||||
}
|
||||
|
||||
// True when the stream handler took the context: the door's write was admitted.
|
||||
public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_owned)
|
||||
((ICollection<KeyValuePair<object, DoorContext>>)_doors).Remove(new KeyValuePair<object, DoorContext>(_identity, _context));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PendingChange
|
||||
{
|
||||
public PendingChange(CultCacheDocumentChange<object> change, DoorContext? door)
|
||||
{
|
||||
Change = change;
|
||||
Door = door;
|
||||
}
|
||||
|
||||
public CultCacheDocumentChange<object> Change { get; }
|
||||
public DoorContext? Door { get; }
|
||||
}
|
||||
|
||||
private sealed class DoorContext
|
||||
{
|
||||
public DoorContext(CultNetShardDescriptor shard) => Shard = shard;
|
||||
|
||||
public CultNetShardDescriptor Shard { get; }
|
||||
public CultNetDatabaseChangeKind? Kind { get; set; }
|
||||
public bool NoLog { get; set; }
|
||||
public bool Removal { get; set; }
|
||||
public CultNetDocumentPutRawMessage? Put { get; set; }
|
||||
public CultNetDocumentDeleteMessage? Delete { get; set; }
|
||||
public CultNetShardLogEntryMessage? Replica { get; set; }
|
||||
// The prediction this door's change either records (Predicts) or ends.
|
||||
public string? PredictionKey { get; set; }
|
||||
public bool Predicts { get; set; }
|
||||
}
|
||||
|
||||
private sealed class ReferenceIdentity : IEqualityComparer<object>
|
||||
{
|
||||
public static readonly ReferenceIdentity Instance = new();
|
||||
|
||||
bool IEqualityComparer<object>.Equals(object? x, object? y) => ReferenceEquals(x, y);
|
||||
|
||||
int IEqualityComparer<object>.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value);
|
||||
}
|
||||
|
||||
private CultRecordKey GetTrackedKey(object document, Type documentType)
|
||||
|
|
@ -1240,16 +1349,6 @@ namespace GameCult.Networking
|
|||
}
|
||||
}
|
||||
|
||||
private void RemoveTrackedDocument(Type documentType, CultRecordKey key)
|
||||
{
|
||||
_cache.Remove(key);
|
||||
}
|
||||
|
||||
private void Publish<T>(CultNetDatabaseChange<T> change) where T : class
|
||||
{
|
||||
_changes.OnNext(change);
|
||||
}
|
||||
|
||||
private sealed class TrackedShardDocument
|
||||
{
|
||||
public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document)
|
||||
|
|
@ -1275,6 +1374,8 @@ namespace GameCult.Networking
|
|||
object? previousDocument,
|
||||
CultNetShardLogEntryMessage? wireEntry = null)
|
||||
{
|
||||
lock (_logGate)
|
||||
{
|
||||
var sequence = NextMutationLogSequence(shard.ShardId);
|
||||
var committedAt = DateTimeOffset.UtcNow.ToString("O");
|
||||
var entry = new CultNetShardMutationLogEntry(
|
||||
|
|
@ -1292,6 +1393,7 @@ namespace GameCult.Networking
|
|||
: wireEntry ?? ToLogEntryMessage(entry);
|
||||
RecordMutationLogEntry(entry, storedWireEntry);
|
||||
_lastWriteSequence[(schemaId, key.Value)] = sequence;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -1299,8 +1401,11 @@ namespace GameCult.Networking
|
|||
/// never been committed through this database. The order the CultNet selection evaluator
|
||||
/// pages by (docs/cultnet-selection-cut.md, section 2).
|
||||
/// </summary>
|
||||
public long? LastWriteSequence(string schemaId, CultRecordKey key) =>
|
||||
_lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null;
|
||||
public long? LastWriteSequence(string schemaId, CultRecordKey key)
|
||||
{
|
||||
lock (_logGate)
|
||||
return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The current write-sequence watermark across every shard this database holds, 0 when nothing
|
||||
|
|
@ -1311,7 +1416,11 @@ namespace GameCult.Networking
|
|||
/// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to
|
||||
/// while the page's own shard sat still.
|
||||
/// </summary>
|
||||
public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max();
|
||||
public ulong CurrentAsOf()
|
||||
{
|
||||
lock (_logGate)
|
||||
return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit
|
||||
|
|
@ -1325,7 +1434,8 @@ namespace GameCult.Networking
|
|||
public ulong CurrentAsOf(string shardId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
|
||||
return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL;
|
||||
lock (_logGate)
|
||||
return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL;
|
||||
}
|
||||
|
||||
private void RecordMutationLogEntry(
|
||||
|
|
@ -1453,8 +1563,6 @@ namespace GameCult.Networking
|
|||
|
||||
var key = new CultRecordKey(message.Document.RecordKey);
|
||||
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
|
||||
var previous = _cache.Get(key);
|
||||
var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false);
|
||||
var kind = ChangeKindFromWire(entry.ChangeKind);
|
||||
if (kind == CultNetDatabaseChangeKind.Removed)
|
||||
{
|
||||
|
|
@ -1463,35 +1571,17 @@ namespace GameCult.Networking
|
|||
}
|
||||
|
||||
var predictionKey = PredictionKey(descriptor.SchemaId, key);
|
||||
if (_predictedDocuments.Remove(predictionKey))
|
||||
var reconcilesPrediction = HasPrediction(predictionKey);
|
||||
if (reconcilesPrediction)
|
||||
{
|
||||
kind = CultNetDatabaseChangeKind.Reconciled;
|
||||
}
|
||||
|
||||
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
|
||||
shard.ShardId,
|
||||
shard.Epoch,
|
||||
entry.Sequence,
|
||||
entry.CommittedAt,
|
||||
kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind,
|
||||
descriptor.SchemaId,
|
||||
key,
|
||||
document,
|
||||
previous),
|
||||
entry);
|
||||
// R-Q: a replica apply is a commit path too - the last-write sequence this row now carries
|
||||
// is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets
|
||||
// it, so the selection evaluator's order and cursor agree whether a row's most recent write
|
||||
// landed locally or arrived through replication.
|
||||
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
|
||||
PublishUntyped(
|
||||
descriptor.DocumentType,
|
||||
kind,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document,
|
||||
previous);
|
||||
// The replica's log records the primary's own entry under the primary's own sequence, in the same hold that
|
||||
// admits the change; the stream handler does both.
|
||||
var document = _documents.DeserializeRawDocument(message.Document);
|
||||
using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = reconcilesPrediction ? predictionKey : null }))
|
||||
await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void ApplyCommittedDelete(
|
||||
|
|
@ -1510,33 +1600,27 @@ namespace GameCult.Networking
|
|||
|
||||
var key = new CultRecordKey(message.RecordKey);
|
||||
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId);
|
||||
var admitted = false;
|
||||
var previous = _cache.Get(key);
|
||||
if (previous != null)
|
||||
{
|
||||
using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true });
|
||||
_cache.Remove(key);
|
||||
admitted = door.Consumed;
|
||||
}
|
||||
|
||||
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
|
||||
shard.ShardId,
|
||||
shard.Epoch,
|
||||
entry.Sequence,
|
||||
entry.CommittedAt,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
descriptor.SchemaId,
|
||||
key,
|
||||
document: null,
|
||||
previousDocument: previous),
|
||||
entry);
|
||||
// R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too.
|
||||
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
|
||||
PublishUntyped(
|
||||
descriptor.DocumentType,
|
||||
CultNetDatabaseChangeKind.Removed,
|
||||
key,
|
||||
descriptor.SchemaId,
|
||||
shard,
|
||||
document: null,
|
||||
previousDocument: previous);
|
||||
if (!admitted)
|
||||
{
|
||||
// The key is not here, so nothing was admitted to log. This replica's log must still hold every entry of the
|
||||
// primary's, or a replica chained on it sees a gap.
|
||||
lock (_logGate)
|
||||
{
|
||||
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
|
||||
shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed,
|
||||
descriptor.SchemaId, key, document: null, previousDocument: null), entry);
|
||||
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry)
|
||||
|
|
|
|||
347
tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs
Normal file
347
tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
public sealed class CultNetPublicationOrderTests
|
||||
{
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
|
||||
private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text;
|
||||
|
||||
// A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed
|
||||
// (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile.
|
||||
[Test]
|
||||
public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var entered = new ManualResetEventSlim();
|
||||
var release = new ManualResetEventSlim();
|
||||
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "A")
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(Wait);
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache);
|
||||
var seen = new List<string?>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); });
|
||||
var key = new CultRecordKey("soul:order");
|
||||
|
||||
var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult());
|
||||
Assert.That(entered.Wait(Wait), Is.True);
|
||||
await database.PutAsync(key, Note("B"));
|
||||
release.Set();
|
||||
await a;
|
||||
|
||||
var shard = database.Shards[0].ShardId;
|
||||
var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last();
|
||||
|
||||
// A replica built from the log.
|
||||
var replicaCache = new CultCache();
|
||||
var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)]
|
||||
});
|
||||
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = shard,
|
||||
ShardEpoch = database.Shards[0].Epoch,
|
||||
Entries = database.GetMutationLogMessages(shard).ToArray()
|
||||
});
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B");
|
||||
Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write");
|
||||
Assert.That(replicaCache.Get<NetworkSchemaNote>(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary");
|
||||
Assert.That(seen.Last(), Is.EqualTo("B"), "a subscriber's final state for the key");
|
||||
});
|
||||
}
|
||||
|
||||
// Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache.
|
||||
[Test]
|
||||
public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing()
|
||||
{
|
||||
var divergent = 0;
|
||||
const int rounds = 300;
|
||||
for (var round = 0; round < rounds; round++)
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("soul:hammer");
|
||||
using var start = new Barrier(2);
|
||||
Parallel.For(0, 2, writer =>
|
||||
{
|
||||
start.SignalAndWait();
|
||||
for (var i = 0; i < 40; i++)
|
||||
cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult();
|
||||
});
|
||||
var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last();
|
||||
if (!ReferenceEquals(last.Document, cache.Get<NetworkSchemaNote>(key)))
|
||||
divergent++;
|
||||
}
|
||||
|
||||
TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}");
|
||||
Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache");
|
||||
}
|
||||
|
||||
// A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in
|
||||
// the window before the door's own change is handled, consumes the door's context.
|
||||
[Test]
|
||||
public async Task S2_ARemovalNeverWearsAReconcilingPutsContext()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var entered = new ManualResetEventSlim();
|
||||
var release = new ManualResetEventSlim();
|
||||
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "authoritative")
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(Wait);
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:reconcile");
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text)));
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
var a = Task.Run(() => database.ApplyPutAsync(message));
|
||||
Assert.That(entered.Wait(Wait), Is.True);
|
||||
cache.Remove(key);
|
||||
release.Set();
|
||||
await a;
|
||||
|
||||
var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1),
|
||||
"the removal is published as a removal: " + string.Join(", ", published));
|
||||
Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text),
|
||||
Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document");
|
||||
Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True,
|
||||
"no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document))));
|
||||
});
|
||||
}
|
||||
|
||||
// A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not
|
||||
// hold (B removed it locally, a supported write) must still keep B's log contiguous for C.
|
||||
[Test]
|
||||
public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey()
|
||||
{
|
||||
const string shardId = "chain";
|
||||
var schemaId = new CultCache().Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])]
|
||||
});
|
||||
var p = Make(new CultCache(), true);
|
||||
var bCache = new CultCache();
|
||||
var b = Make(bCache, false);
|
||||
var c = Make(new CultCache(), false);
|
||||
async Task Pull(CultNetDatabase from, CultNetDatabase to)
|
||||
{
|
||||
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = shardId,
|
||||
ShardEpoch = 2,
|
||||
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray()
|
||||
});
|
||||
}
|
||||
|
||||
var k = new CultRecordKey("soul:chain:k");
|
||||
await p.PutAsync(k, Note("x"));
|
||||
await Pull(p, b);
|
||||
await Pull(b, c);
|
||||
bCache.Remove(k);
|
||||
await p.DeleteAsync<NetworkSchemaNote>(k);
|
||||
await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y"));
|
||||
await Pull(p, b);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }),
|
||||
"B's log holds every primary entry it applied");
|
||||
Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap");
|
||||
});
|
||||
}
|
||||
|
||||
// A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction.
|
||||
[Test]
|
||||
public async Task S4_APredictionMadeInReactionToAReconcileSurvives()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:repredict");
|
||||
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
|
||||
var repredicted = false;
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
kinds.Enqueue(change.Kind);
|
||||
if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted)
|
||||
{
|
||||
repredicted = true;
|
||||
database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult();
|
||||
}
|
||||
});
|
||||
CultNetDocumentPutRawMessage Put(string text)
|
||||
{
|
||||
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
return message;
|
||||
}
|
||||
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
await database.ApplyPutAsync(Put("auth1"));
|
||||
await database.ApplyPutAsync(Put("auth2"));
|
||||
|
||||
Assert.That(kinds, Is.EqualTo(new[]
|
||||
{
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
CultNetDatabaseChangeKind.Reconciled,
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
CultNetDatabaseChangeKind.Reconciled
|
||||
}));
|
||||
}
|
||||
// Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A
|
||||
// refused prediction leaves nothing behind that a later write of the same instance would wear.
|
||||
[Test]
|
||||
public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<CultNetDocumentRegistryVariantTests.VariantWireFixture>(registry));
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = documents,
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var baseKey = new CultRecordKey("soul-refuse-base");
|
||||
var variantKey = new CultRecordKey("soul-refuse-variant");
|
||||
await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 },
|
||||
new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<CultNetDocumentRegistryVariantTests.VariantWireFixture>(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v")
|
||||
});
|
||||
var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 };
|
||||
Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException);
|
||||
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
|
||||
using var subscriber = database.Watch<CultNetDocumentRegistryVariantTests.VariantWireFixture>().Subscribe(change => kinds.Enqueue(change.Kind));
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
|
||||
await cache.UpsertAsync(instance, new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(new CultRecordKey("soul-refuse-other")));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1));
|
||||
});
|
||||
}
|
||||
|
||||
// Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a
|
||||
// durable log store can observe even though the disposed Subject hides publication.
|
||||
[Test]
|
||||
public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog()
|
||||
{
|
||||
var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N"));
|
||||
var store = new CultNetFileShardMutationLogStore(root);
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store });
|
||||
var shard = database.Shards[0].ShardId;
|
||||
await cache.UpsertAsync(Note("before"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:before")));
|
||||
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
|
||||
database.Dispose();
|
||||
|
||||
try { await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:after"))); }
|
||||
catch { /* a throwing handler would also be a failure below */ }
|
||||
|
||||
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
|
||||
System.IO.Directory.Delete(root, true);
|
||||
}
|
||||
|
||||
// The log entry exists before its change is published: a subscriber that reads the log from its handler finds it.
|
||||
[Test]
|
||||
public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("soul:log-first");
|
||||
var loggedWhenPublished = new ConcurrentQueue<bool>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value)));
|
||||
|
||||
await database.PutAsync(key, Note("one"));
|
||||
cache.Remove(key);
|
||||
|
||||
Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true }));
|
||||
}
|
||||
|
||||
// An observer that mutates the admitted instance in place and upserts it again writes a second change of the same
|
||||
// instance. It is published as its own update and never wears the context of the door's change, even when it
|
||||
// reaches the database first.
|
||||
[Test]
|
||||
public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "authoritative")
|
||||
{
|
||||
change.Document.Text = "normalized";
|
||||
cache.UpsertAsync(change.Document, new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:in-place");
|
||||
var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text)));
|
||||
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
await database.ApplyPutAsync(message);
|
||||
|
||||
Assert.That(seen, Is.EqualTo(new (CultNetDatabaseChangeKind, string?)[]
|
||||
{
|
||||
(CultNetDatabaseChangeKind.Predicted, null),
|
||||
(CultNetDatabaseChangeKind.Reconciled, "predicted"),
|
||||
(CultNetDatabaseChangeKind.Updated, "normalized")
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
429
tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs
Normal file
429
tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using GameCult.Caching.MessagePack;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
// The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the
|
||||
// mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every
|
||||
// other change in the hold publishes from the cache.
|
||||
public sealed class CultNetPublishesCommitsTests
|
||||
{
|
||||
private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) =>
|
||||
wire.Changes.Where(change => KeyOf(change) == key).ToArray();
|
||||
|
||||
private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey;
|
||||
|
||||
private sealed class Wire : IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cancellation = new();
|
||||
private readonly Thread _thread;
|
||||
private readonly RudpCultNetSchemaServer _server;
|
||||
private readonly CultNetDatabaseSubscriptionServer _subscriptions;
|
||||
private readonly ICultNetSchemaClient _client;
|
||||
public ConcurrentQueue<CultNetDatabaseChangeRawMessage> Changes { get; } = new();
|
||||
public int Snapshots;
|
||||
public ConcurrentQueue<CultNetErrorMessage> Errors { get; } = new();
|
||||
|
||||
private readonly CultNetDatabase _database;
|
||||
|
||||
public Wire(CultNetDatabase database)
|
||||
{
|
||||
_database = database;
|
||||
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
socket.ReceiveTimeout = 20;
|
||||
_server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket });
|
||||
_subscriptions = new CultNetDatabaseSubscriptionServer(_server, database);
|
||||
_thread = new Thread(() =>
|
||||
{
|
||||
while (!_cancellation.IsCancellationRequested)
|
||||
{
|
||||
_ = _server.PollOnceAsync().GetAwaiter().GetResult();
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}) { IsBackground = true };
|
||||
_thread.Start();
|
||||
_client = CultNetSchemaClients.CreateRudp("publish-client");
|
||||
_client.OnCultNet<CultNetDatabaseChangeRawMessage>(Changes.Enqueue);
|
||||
_client.OnCultNet<CultNetErrorMessage>(Errors.Enqueue);
|
||||
_client.OnCultNet<CultNetSnapshotResponseRawMessage>(_ => Interlocked.Increment(ref Snapshots));
|
||||
_client.Connect("127.0.0.1", _server.LocalEndPoint.Port);
|
||||
}
|
||||
|
||||
public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes)
|
||||
{
|
||||
await Until(() => _client.Connected);
|
||||
_client.SendCultNet(new CultNetDatabaseSubscribeMessage
|
||||
{
|
||||
MessageId = "subscribe",
|
||||
SubscriptionId = "all",
|
||||
SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(),
|
||||
IncludeSnapshot = includeSnapshot
|
||||
});
|
||||
await Until(() => Snapshots > 0 || !Errors.IsEmpty);
|
||||
Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cancellation.Cancel();
|
||||
_thread.Join();
|
||||
_client.Dispose();
|
||||
_subscriptions.Dispose();
|
||||
_server.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Until(Func<bool> condition)
|
||||
{
|
||||
var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTimeOffset.UtcNow > deadline)
|
||||
Assert.Fail("Condition was not satisfied before timeout.");
|
||||
await Task.Delay(5);
|
||||
}
|
||||
}
|
||||
|
||||
// One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance
|
||||
// to arrive twice.
|
||||
private static async Task SettleAsync(CultNetDatabase database, Wire wire)
|
||||
{
|
||||
var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N");
|
||||
await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" });
|
||||
await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel));
|
||||
}
|
||||
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
|
||||
[Test]
|
||||
public async Task ACacheCommitAndUpsertReachASubscriberOnce()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:commit");
|
||||
|
||||
cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle<NetworkSchemaNote>(key)));
|
||||
await Until(() => Of(wire, key.Value).Length >= 1);
|
||||
cache.Commit(batch => batch.Remove(key));
|
||||
await Until(() => Of(wire, key.Value).Length >= 2);
|
||||
await cache.UpsertAsync(Note("upserted"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADatabasePutAndDeleteStillPublishExactlyOnce()
|
||||
{
|
||||
var database = new CultNetDatabase(new CultCache());
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:put");
|
||||
|
||||
await database.PutAsync(key, Note("put"));
|
||||
await database.DeleteAsync<NetworkSchemaNote>(key);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:replicated");
|
||||
var message = database.Documents.CreateRawDocumentPutMessage(
|
||||
"put-replicated", new CultRecordHandle<NetworkSchemaNote>(key), Note("remote"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
await database.ApplyPutAsync(message);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALoadRemovalCarriesItsKey()
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc");
|
||||
try
|
||||
{
|
||||
var key = new CultRecordKey("tests:publish:loaded");
|
||||
var writer = new CultCache();
|
||||
writer.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
await writer.UpsertAsync(Note("there"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
writer.FlushAllBackingStores();
|
||||
|
||||
var cache = new CultCache();
|
||||
cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
var database = new CultNetDatabase(cache);
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key), Is.Not.Null);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(true, typeof(NetworkSchemaNote));
|
||||
|
||||
writer.Remove(key);
|
||||
writer.FlushAllBackingStores();
|
||||
await cache.PullAllBackingStoresAsync();
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" }));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(path)) File.Delete(path);
|
||||
if (File.Exists(path + ".lock")) File.Delete(path + ".lock");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big"),
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Power), 9)
|
||||
});
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(VariantWireFixture));
|
||||
var errorsBefore = wire.Errors.Count;
|
||||
|
||||
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
await Until(() => wire.Errors.Count > errorsBefore);
|
||||
await Until(() => Of(wire, baseKey.Value).Length >= 1);
|
||||
|
||||
Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value),
|
||||
"the re-resolved dependent is published, then refused (Q6) by key");
|
||||
Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1));
|
||||
Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record");
|
||||
}
|
||||
|
||||
// An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own.
|
||||
[Test]
|
||||
public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:normalized");
|
||||
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "raw")
|
||||
cache.UpsertAsync(Note("normalized"), new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
|
||||
});
|
||||
|
||||
await database.PutAsync(key, Note("raw"));
|
||||
await Until(() => Of(wire, key.Value).Length >= 2);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("normalized"));
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
|
||||
}
|
||||
|
||||
private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
|
||||
return (cache, documents);
|
||||
}
|
||||
|
||||
private static System.Collections.Generic.List<string> Record(CultNetDatabase database, Func<object, string> describe)
|
||||
{
|
||||
var seen = new System.Collections.Generic.List<string>();
|
||||
database.WatchAllChanges().Subscribe(change => seen.Add(describe(change)));
|
||||
return seen;
|
||||
}
|
||||
|
||||
private static string KeyOfChange(object change) => ((dynamic)change).Key.Value;
|
||||
|
||||
[Test]
|
||||
public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder()
|
||||
{
|
||||
var (cache, documents) = VariantCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
|
||||
});
|
||||
var published = Record(database, KeyOfChange);
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
|
||||
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value),
|
||||
Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
|
||||
}
|
||||
|
||||
private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) =>
|
||||
new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = documents,
|
||||
Shards =
|
||||
[
|
||||
new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId])
|
||||
]
|
||||
});
|
||||
|
||||
[Test]
|
||||
public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally()
|
||||
{
|
||||
var (sourceCache, sourceDocuments) = VariantCache();
|
||||
var (replicaCache, replicaDocuments) = VariantCache();
|
||||
var schemaId = sourceCache.Registry.GetRequired<VariantWireFixture>().SchemaId;
|
||||
var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = sourceDocuments,
|
||||
Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])]
|
||||
});
|
||||
var replica = Replica(replicaCache, replicaDocuments, schemaId);
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
replicaCache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
|
||||
});
|
||||
await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
|
||||
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = "replica-shard",
|
||||
ShardEpoch = 3,
|
||||
Entries = source.GetMutationLogMessages("replica-shard").ToArray()
|
||||
});
|
||||
|
||||
Assert.That(replicaCache.Get<VariantWireFixture>(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved");
|
||||
Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }),
|
||||
"the replica log holds the primary's entry and mints none of its own");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
var replica = Replica(cache, null, schemaId);
|
||||
var published = Record(replica, KeyOfChange);
|
||||
|
||||
await cache.UpsertAsync(Note("local"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:local")));
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" }));
|
||||
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
const int writers = 4, perWriter = 5000;
|
||||
|
||||
Parallel.For(0, writers, writer =>
|
||||
{
|
||||
for (var i = 0; i < perWriter; i++)
|
||||
cache.UpsertAsync(Note("n"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey($"tests:publish:c:{writer}:{i}")))
|
||||
.GetAwaiter().GetResult();
|
||||
});
|
||||
|
||||
var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray();
|
||||
Assert.That(sequences, Has.Length.EqualTo(writers * perWriter));
|
||||
Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("tests:publish:after-delete");
|
||||
await database.PutAsync(key, Note("one"));
|
||||
await database.DeleteAsync<NetworkSchemaNote>(key);
|
||||
var published = Record(database, KeyOfChange);
|
||||
|
||||
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { key.Value }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind),
|
||||
Is.EqualTo(new[]
|
||||
{
|
||||
CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added
|
||||
}));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
var replica = Replica(cache, null, schemaId);
|
||||
var stale = new CultRecordKey("tests:publish:stale");
|
||||
var fresh = new CultRecordKey("tests:publish:fresh");
|
||||
await cache.UpsertAsync(Note("stale"), new CultRecordHandle<NetworkSchemaNote>(stale));
|
||||
var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value);
|
||||
var put = replica.Documents.CreateRawDocumentPutMessage(
|
||||
"snapshot-fresh", new CultRecordHandle<NetworkSchemaNote>(fresh), Note("fresh"));
|
||||
|
||||
await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage
|
||||
{
|
||||
ShardId = "replica-shard",
|
||||
ShardEpoch = 3,
|
||||
Documents = [put.Document!],
|
||||
ShardLogSequence = 5
|
||||
});
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value }));
|
||||
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADisposedDatabaseNoLongerObservesTheCache()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var published = Record(database, KeyOfChange);
|
||||
database.Dispose();
|
||||
|
||||
await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:disposed")));
|
||||
|
||||
Assert.That(published, Is.Empty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests
|
|||
Assert.That(cache.Get(batched), Is.Null);
|
||||
}), Is.True);
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { put }));
|
||||
Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it");
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(batched)?.Text, Is.EqualTo("batched"));
|
||||
var reopened = new CultCache();
|
||||
reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
|
|
@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests
|
|||
|
||||
var cache = new CultCache();
|
||||
var delivering = new ManualResetEventSlim();
|
||||
cache.OnUpdate += (_, _) => delivering.Set();
|
||||
// Subscribed before the database, so it runs before the database's own observer of the same change.
|
||||
using var loadDelivering = cache.Watch<object>().Subscribe(_ => delivering.Set());
|
||||
var database = new CultNetDatabase(cache);
|
||||
var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue