Merge the salvaged CultNet publication branch (queue removed in the next commit)

Cut 4a: brings the publication tests and the door table from
hands/cultnet-publish-commits at 28248ebe. The reorder buffer, drainer and
prediction gate die in 4b.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-29 23:36:55 +02:00
commit 7d945b002d
5 changed files with 1109 additions and 243 deletions

View file

@ -47,10 +47,15 @@ which route wrote it.
- Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a
per-cache, in-memory number assigned when the cache admits the change, under
its gate. It increases in admission order and is not persisted.
`OnUpdate(previous, current)` carries none. Streams CultNet derives from
`OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server,
database-backed Mesh handles) have no ordering guarantee and no stale
protection.
`OnUpdate(previous, current)` carries none. CultNet derives its
streams (`CultNetDatabase.WatchAllChanges`, the subscription server,
database-backed Mesh handles) from `Watch`, so a load, a commit and a variant
dependent's re-resolution all reach subscribers and the mutation log through
one handler over `Watch`; a database write door only hands that handler the
wire message or replicated entry for the change it admitted. That handler releases
changes in cache `Sequence` order, so the log and the stream agree with the
cache. A replica publishes but never mints log sequences. The
streams carry no `Sequence` and have no stale protection.
- `GetWithSequence(key)` returns the document and the cache's current
`Sequence`, read together under the gate: every change with a `Sequence` at
or below it is reflected in the document.

View file

@ -1,7 +1,9 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
@ -472,7 +474,7 @@ namespace GameCult.Networking
private readonly List<CultNetShardDescriptor> _shards;
private readonly List<CultNetClientAuthorityScope> _clientAuthorityScopes;
private readonly ICultNetShardMutationLogStore? _mutationLogStore;
private readonly Dictionary<string, object> _predictedDocuments = new(StringComparer.Ordinal);
private readonly HashSet<string> _predictedDocuments = new(StringComparer.Ordinal);
private readonly Dictionary<string, List<CultNetShardMutationLogEntry>> _mutationLogs =
new(StringComparer.Ordinal);
private readonly Dictionary<string, long> _nextLogSequences = new(StringComparer.Ordinal);
@ -483,6 +485,26 @@ namespace GameCult.Networking
private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence =
new();
private readonly Subject<object> _changes = new();
private readonly IDisposable _cacheChanges;
// What a write door of this database knows about the change it is admitting and the cache does not: the wire
// message it carries, a prediction it reconciles, the replicated log entry it applies. The cache's Watch stream
// is the only publisher and the only logger; a door registers this context under the identity of the change it
// admits (the instance it writes, or the instance it removes) and the stream handler consumes it once, for that
// change alone. A write to the same key by anyone else is a different instance and gets no context.
private readonly ConcurrentDictionary<object, DoorContext> _doors = new(ReferenceIdentity.Instance);
// One gate over the mutation log, its sequence counters and the last-write map. Publication stays outside it: a
// subscriber may write the cache from its handler.
private readonly object _logGate = new();
// The cache admits a change to a key under its gate, but each thread reaches this handler after leaving it, in any
// order. Changes are therefore released strictly in cache Sequence order: a change ahead of a gap is parked until the
// gap fills, and one thread at a time drains the ready queue, outside every lock, so a subscriber that writes the
// cache from its handler queues behind the change it is handling instead of running inside it.
private readonly SortedDictionary<long, PendingChange> _parked = new();
private readonly Queue<PendingChange> _ready = new();
private long _nextSequence;
private bool _draining;
// Predictions this runtime made and has not seen reconciled. Every read and write takes _predictionGate.
private readonly object _predictionGate = new();
private bool _disposed;
/// <summary>
@ -502,7 +524,12 @@ namespace GameCult.Networking
_clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty<CultNetClientAuthorityScope>()).ToList();
CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random();
InitializeLogSequencesFromStore();
_cache.OnUpdate += PublishCacheUpdate;
lock (_logGate)
{
_cacheChanges = _cache.Watch<object>().Subscribe(OnCacheChange);
// Changes admitted before this read are released as they arrive; the next one is the first this handler waits for.
_nextSequence = _cache.GetWithSequence(new CultRecordKey(string.Empty)).Sequence + 1;
}
}
/// <summary>
@ -557,18 +584,21 @@ namespace GameCult.Networking
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (!_mutationLogs.TryGetValue(shardId, out var entries))
lock (_logGate)
{
return Array.Empty<CultNetShardMutationLogEntry>();
}
if (!_mutationLogs.TryGetValue(shardId, out var entries))
{
return Array.Empty<CultNetShardMutationLogEntry>();
}
var query = entries.Where(entry => entry.Sequence > afterSequence);
if (limit.HasValue)
{
query = query.Take(limit.Value);
}
var query = entries.Where(entry => entry.Sequence > afterSequence);
if (limit.HasValue)
{
query = query.Take(limit.Value);
}
return query.ToArray();
return query.ToArray();
}
}
/// <summary>
@ -700,39 +730,17 @@ namespace GameCult.Networking
var local = GetLocalShardDocuments(shard).ToArray();
foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value)))
{
RemoveTrackedDocument(existing.DocumentType, existing.Key);
PublishUntyped(
existing.DocumentType,
CultNetDatabaseChangeKind.Removed,
existing.Key,
existing.SchemaId,
shard,
document: null,
previousDocument: existing.Document);
using (Door(existing.Document, new DoorContext(shard) { NoLog = true }))
_cache.Remove(existing.Key);
}
foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document)))
{
var key = new CultRecordKey(document.RecordKey);
var descriptor = _documents.ResolveDescriptorForRawDocument(document);
var previous = _cache.Get(key);
var applied = await _documents.ApplyRawDocumentPutMessageAsync(
_cache,
new CultNetDocumentPutRawMessage
{
MessageId = response.MessageId,
Document = document,
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch
}).ConfigureAwait(false);
PublishUntyped(
descriptor.DocumentType,
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
key,
descriptor.SchemaId,
shard,
applied,
previous);
var applied = _documents.DeserializeRawDocument(document);
using (Door(applied, new DoorContext(shard) { NoLog = true }))
await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false);
}
var sequence = response.ShardLogSequence ?? 0;
@ -846,27 +854,7 @@ namespace GameCult.Networking
var shard = ResolveShardInternal(descriptor, key);
EnsurePrimary(shard, descriptor.SchemaId, key);
var previous = _cache.Get<T>(key);
var handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
void PublishCommittedPut()
{
AppendMutationLogEntry(
shard,
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
key,
descriptor.SchemaId,
document,
previous);
Publish(new CultNetDatabaseChange<T>(
previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated,
key,
descriptor.SchemaId,
shard,
document,
previous));
}
PublishCommittedPut();
return handle;
return await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
}
/// <summary>
@ -881,16 +869,15 @@ namespace GameCult.Networking
var shard = ResolveShardInternal(descriptor, key);
EnsureClientAuthority(descriptor, key);
var previous = _cache.Get<T>(key);
var handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
_predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document;
Publish(new CultNetDatabaseChange<T>(
CultNetDatabaseChangeKind.Predicted,
key,
descriptor.SchemaId,
shard,
document,
previous));
CultRecordHandle<T> handle;
using (Door(document, new DoorContext(shard)
{
Kind = CultNetDatabaseChangeKind.Predicted,
NoLog = true,
PredictionKey = PredictionKey(descriptor.SchemaId, key),
Predicts = true
}))
handle = await _cache.UpsertAsync(document, new CultRecordHandle<T>(key)).ConfigureAwait(false);
return handle;
}
@ -904,30 +891,7 @@ namespace GameCult.Networking
var shard = ResolveShardInternal(descriptor, key);
EnsurePrimary(shard, descriptor.SchemaId, key);
var previous = _cache.Get<T>(key);
if (previous != null)
{
_cache.Remove(new CultRecordHandle<T>(key));
void PublishCommittedDelete()
{
AppendMutationLogEntry(
shard,
CultNetDatabaseChangeKind.Removed,
key,
descriptor.SchemaId,
document: null,
previousDocument: previous);
Publish(new CultNetDatabaseChange<T>(
CultNetDatabaseChangeKind.Removed,
key,
descriptor.SchemaId,
shard,
document: null,
previousDocument: previous));
}
PublishCommittedDelete();
}
_cache.Remove(new CultRecordHandle<T>(key));
return Task.CompletedTask;
}
@ -947,42 +911,16 @@ namespace GameCult.Networking
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
var shard = ResolveShardInternal(descriptor, key);
EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch);
var previous = _cache.Get(key);
var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false);
var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated;
var document = _documents.DeserializeRawDocument(message.Document);
var predictionKey = PredictionKey(descriptor.SchemaId, key);
var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey);
if (reconcilesPrediction)
var reconcilesPrediction = HasPrediction(predictionKey);
using (Door(document, new DoorContext(shard)
{
kind = CultNetDatabaseChangeKind.Reconciled;
}
void PublishCommittedPut()
{
if (reconcilesPrediction)
_predictedDocuments.Remove(predictionKey);
AppendMutationLogEntry(
shard,
kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind,
key,
descriptor.SchemaId,
document,
previous,
new CultNetShardLogEntryMessage
{
ChangeKind = previous == null ? "added" : "updated",
Put = message
});
PublishUntyped(
descriptor.DocumentType,
kind,
key,
descriptor.SchemaId,
shard,
document,
previous);
}
PublishCommittedPut();
Put = message,
Kind = reconcilesPrediction ? CultNetDatabaseChangeKind.Reconciled : null,
PredictionKey = reconcilesPrediction ? predictionKey : null
}))
await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false);
return document;
}
@ -1004,31 +942,8 @@ namespace GameCult.Networking
return Task.CompletedTask;
}
_cache.Remove(key);
void PublishCommittedDelete()
{
AppendMutationLogEntry(
shard,
CultNetDatabaseChangeKind.Removed,
key,
descriptor.SchemaId,
document: null,
previousDocument: previous,
wireEntry: new CultNetShardLogEntryMessage
{
ChangeKind = "removed",
Delete = message
});
PublishUntyped(
descriptor.DocumentType,
CultNetDatabaseChangeKind.Removed,
key,
descriptor.SchemaId,
shard,
document: null,
previousDocument: previous);
}
PublishCommittedDelete();
using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true }))
_cache.Remove(key);
return Task.CompletedTask;
}
@ -1154,40 +1069,234 @@ namespace GameCult.Networking
}
_disposed = true;
_cache.OnUpdate -= PublishCacheUpdate;
_cacheChanges.Dispose();
_changes.Dispose();
}
private void PublishCacheUpdate(object? previous, object? current)
// The cache is the one owner of "a change was committed", and this is its one publisher and one logger: every admitted
// change reaches subscribers and, when this runtime is the shard's primary, the mutation log, exactly once, in cache
// Sequence order. A replica publishes what it admits but never mints log sequences: its log holds only the primary's
// own entries, which the replica door hands over.
private void OnCacheChange(CultCacheDocumentChange<object> change)
{
var document = current ?? previous;
if (document == null)
var pending = new PendingChange(change, TakeDoor(change));
lock (_logGate)
{
if (_disposed)
{
return;
}
if (change.Sequence < _nextSequence)
{
_ready.Enqueue(pending);
}
else
{
_parked[change.Sequence] = pending;
while (_parked.Remove(_nextSequence, out var next))
{
_ready.Enqueue(next);
_nextSequence++;
}
}
if (_draining)
{
return;
}
_draining = true;
}
try
{
while (true)
{
PendingChange? item;
lock (_logGate)
{
if (_disposed || !_ready.TryDequeue(out item))
{
_draining = false;
return;
}
}
Deliver(item);
}
}
catch
{
lock (_logGate)
_draining = false;
throw;
}
}
private void Deliver(PendingChange item)
{
var change = item.Change;
var door = item.Door;
var identity = change.Document ?? change.PreviousDocument;
if (identity == null)
{
return;
}
var documentType = document.GetType();
var documentType = identity.GetType();
var descriptor = _cache.Registry.GetRequired(documentType);
var key = current != null
? GetTrackedKey(current, documentType)
: new CultRecordKey(string.Empty);
var shard = ResolveShardInternal(descriptor, key);
var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType);
var change = Activator.CreateInstance(
changeType,
current == null ? CultNetDatabaseChangeKind.Removed :
previous == null ? CultNetDatabaseChangeKind.Added :
CultNetDatabaseChangeKind.Updated,
key,
descriptor.SchemaId,
shard,
current,
previous,
null);
if (change != null)
var shard = door?.Shard ?? ResolveShardInternal(descriptor, change.Key);
var kind = door?.Kind ?? change.Kind switch
{
_changes.OnNext(change);
CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed,
CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added,
_ => CultNetDatabaseChangeKind.Updated
};
var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind;
if (door?.PredictionKey is { } predictionKey)
{
lock (_predictionGate)
{
if (door.Predicts)
_predictedDocuments.Add(predictionKey);
else
_predictedDocuments.Remove(predictionKey);
}
}
if (door?.Replica is { } replicated)
{
lock (_logGate)
{
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
shard.ShardId,
shard.Epoch,
replicated.Sequence,
replicated.CommittedAt,
logKind,
descriptor.SchemaId,
change.Key,
change.Document,
change.PreviousDocument),
replicated);
_lastWriteSequence[(descriptor.SchemaId, change.Key.Value)] = replicated.Sequence;
}
}
else if (door?.NoLog != true && shard.IsPrimary)
{
AppendMutationLogEntry(
shard,
logKind,
change.Key,
descriptor.SchemaId,
change.Document,
change.PreviousDocument,
door?.Put != null
? new CultNetShardLogEntryMessage
{
ChangeKind = change.Kind == CultCacheDocumentChangeKind.Added ? "added" : "updated",
Put = door.Put
}
: door?.Delete != null
? new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = door.Delete }
: null);
}
PublishUntyped(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument);
}
private bool HasPrediction(string predictionKey)
{
lock (_predictionGate)
return _predictedDocuments.Contains(predictionKey);
}
// A door's context belongs to one specific change, recognised by shape: a put door wrote an instance no cache had
// held, so its change carries that instance as the new document and never as the previous one; a removal door
// removes an instance, so its change carries it as the previous document and no new one. A plain removal of an
// instance a put door admitted, or an observer re-upserting that instance in place, is neither, and gets no context.
private DoorContext? TakeDoor(CultCacheDocumentChange<object> change)
{
var identity = change.Document ?? change.PreviousDocument;
if (identity == null || !_doors.TryGetValue(identity, out var door))
{
return null;
}
var matches = change.Document == null
? door.Removal
: !door.Removal && !ReferenceEquals(change.PreviousDocument, change.Document);
return matches && ((ICollection<KeyValuePair<object, DoorContext>>)_doors)
.Remove(new KeyValuePair<object, DoorContext>(identity, door))
? door
: null;
}
private DoorRegistration Door(object identity, DoorContext context) =>
new(_doors, identity, context, _doors.TryAdd(identity, context));
// Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind.
private readonly struct DoorRegistration : IDisposable
{
private readonly ConcurrentDictionary<object, DoorContext> _doors;
private readonly object _identity;
private readonly DoorContext _context;
private readonly bool _owned;
public DoorRegistration(ConcurrentDictionary<object, DoorContext> doors, object identity, DoorContext context, bool owned)
{
_doors = doors;
_identity = identity;
_context = context;
_owned = owned;
}
// True when the stream handler took the context: the door's write was admitted.
public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context);
public void Dispose()
{
if (_owned)
((ICollection<KeyValuePair<object, DoorContext>>)_doors).Remove(new KeyValuePair<object, DoorContext>(_identity, _context));
}
}
private sealed class PendingChange
{
public PendingChange(CultCacheDocumentChange<object> change, DoorContext? door)
{
Change = change;
Door = door;
}
public CultCacheDocumentChange<object> Change { get; }
public DoorContext? Door { get; }
}
private sealed class DoorContext
{
public DoorContext(CultNetShardDescriptor shard) => Shard = shard;
public CultNetShardDescriptor Shard { get; }
public CultNetDatabaseChangeKind? Kind { get; set; }
public bool NoLog { get; set; }
public bool Removal { get; set; }
public CultNetDocumentPutRawMessage? Put { get; set; }
public CultNetDocumentDeleteMessage? Delete { get; set; }
public CultNetShardLogEntryMessage? Replica { get; set; }
// The prediction this door's change either records (Predicts) or ends.
public string? PredictionKey { get; set; }
public bool Predicts { get; set; }
}
private sealed class ReferenceIdentity : IEqualityComparer<object>
{
public static readonly ReferenceIdentity Instance = new();
bool IEqualityComparer<object>.Equals(object? x, object? y) => ReferenceEquals(x, y);
int IEqualityComparer<object>.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value);
}
private CultRecordKey GetTrackedKey(object document, Type documentType)
@ -1240,16 +1349,6 @@ namespace GameCult.Networking
}
}
private void RemoveTrackedDocument(Type documentType, CultRecordKey key)
{
_cache.Remove(key);
}
private void Publish<T>(CultNetDatabaseChange<T> change) where T : class
{
_changes.OnNext(change);
}
private sealed class TrackedShardDocument
{
public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document)
@ -1275,6 +1374,8 @@ namespace GameCult.Networking
object? previousDocument,
CultNetShardLogEntryMessage? wireEntry = null)
{
lock (_logGate)
{
var sequence = NextMutationLogSequence(shard.ShardId);
var committedAt = DateTimeOffset.UtcNow.ToString("O");
var entry = new CultNetShardMutationLogEntry(
@ -1292,6 +1393,7 @@ namespace GameCult.Networking
: wireEntry ?? ToLogEntryMessage(entry);
RecordMutationLogEntry(entry, storedWireEntry);
_lastWriteSequence[(schemaId, key.Value)] = sequence;
}
}
/// <summary>
@ -1299,8 +1401,11 @@ namespace GameCult.Networking
/// never been committed through this database. The order the CultNet selection evaluator
/// pages by (docs/cultnet-selection-cut.md, section 2).
/// </summary>
public long? LastWriteSequence(string schemaId, CultRecordKey key) =>
_lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null;
public long? LastWriteSequence(string schemaId, CultRecordKey key)
{
lock (_logGate)
return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null;
}
/// <summary>
/// The current write-sequence watermark across every shard this database holds, 0 when nothing
@ -1311,7 +1416,11 @@ namespace GameCult.Networking
/// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to
/// while the page's own shard sat still.
/// </summary>
public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max();
public ulong CurrentAsOf()
{
lock (_logGate)
return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max();
}
/// <summary>
/// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit
@ -1325,7 +1434,8 @@ namespace GameCult.Networking
public ulong CurrentAsOf(string shardId)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL;
lock (_logGate)
return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL;
}
private void RecordMutationLogEntry(
@ -1453,8 +1563,6 @@ namespace GameCult.Networking
var key = new CultRecordKey(message.Document.RecordKey);
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
var previous = _cache.Get(key);
var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false);
var kind = ChangeKindFromWire(entry.ChangeKind);
if (kind == CultNetDatabaseChangeKind.Removed)
{
@ -1463,35 +1571,17 @@ namespace GameCult.Networking
}
var predictionKey = PredictionKey(descriptor.SchemaId, key);
if (_predictedDocuments.Remove(predictionKey))
var reconcilesPrediction = HasPrediction(predictionKey);
if (reconcilesPrediction)
{
kind = CultNetDatabaseChangeKind.Reconciled;
}
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
shard.ShardId,
shard.Epoch,
entry.Sequence,
entry.CommittedAt,
kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind,
descriptor.SchemaId,
key,
document,
previous),
entry);
// R-Q: a replica apply is a commit path too - the last-write sequence this row now carries
// is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets
// it, so the selection evaluator's order and cursor agree whether a row's most recent write
// landed locally or arrived through replication.
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
PublishUntyped(
descriptor.DocumentType,
kind,
key,
descriptor.SchemaId,
shard,
document,
previous);
// The replica's log records the primary's own entry under the primary's own sequence, in the same hold that
// admits the change; the stream handler does both.
var document = _documents.DeserializeRawDocument(message.Document);
using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = reconcilesPrediction ? predictionKey : null }))
await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false);
}
private void ApplyCommittedDelete(
@ -1510,33 +1600,27 @@ namespace GameCult.Networking
var key = new CultRecordKey(message.RecordKey);
var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId);
var admitted = false;
var previous = _cache.Get(key);
if (previous != null)
{
using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true });
_cache.Remove(key);
admitted = door.Consumed;
}
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
shard.ShardId,
shard.Epoch,
entry.Sequence,
entry.CommittedAt,
CultNetDatabaseChangeKind.Removed,
descriptor.SchemaId,
key,
document: null,
previousDocument: previous),
entry);
// R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too.
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
PublishUntyped(
descriptor.DocumentType,
CultNetDatabaseChangeKind.Removed,
key,
descriptor.SchemaId,
shard,
document: null,
previousDocument: previous);
if (!admitted)
{
// The key is not here, so nothing was admitted to log. This replica's log must still hold every entry of the
// primary's, or a replica chained on it sees a gap.
lock (_logGate)
{
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed,
descriptor.SchemaId, key, document: null, previousDocument: null), entry);
_lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence;
}
}
}
private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry)

View file

@ -0,0 +1,347 @@
#nullable enable
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
public sealed class CultNetPublicationOrderTests
{
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text;
// A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed
// (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile.
[Test]
public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder()
{
var cache = new CultCache();
var entered = new ManualResetEventSlim();
var release = new ManualResetEventSlim();
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "A")
{
entered.Set();
release.Wait(Wait);
}
});
var database = new CultNetDatabase(cache);
var seen = new List<string?>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); });
var key = new CultRecordKey("soul:order");
var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult());
Assert.That(entered.Wait(Wait), Is.True);
await database.PutAsync(key, Note("B"));
release.Set();
await a;
var shard = database.Shards[0].ShardId;
var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last();
// A replica built from the log.
var replicaCache = new CultCache();
var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)]
});
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = shard,
ShardEpoch = database.Shards[0].Epoch,
Entries = database.GetMutationLogMessages(shard).ToArray()
});
Assert.Multiple(() =>
{
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B");
Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write");
Assert.That(replicaCache.Get<NetworkSchemaNote>(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary");
Assert.That(seen.Last(), Is.EqualTo("B"), "a subscriber's final state for the key");
});
}
// Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache.
[Test]
public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing()
{
var divergent = 0;
const int rounds = 300;
for (var round = 0; round < rounds; round++)
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("soul:hammer");
using var start = new Barrier(2);
Parallel.For(0, 2, writer =>
{
start.SignalAndWait();
for (var i = 0; i < 40; i++)
cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult();
});
var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last();
if (!ReferenceEquals(last.Document, cache.Get<NetworkSchemaNote>(key)))
divergent++;
}
TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}");
Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache");
}
// A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in
// the window before the door's own change is handled, consumes the door's context.
[Test]
public async Task S2_ARemovalNeverWearsAReconcilingPutsContext()
{
var cache = new CultCache();
var entered = new ManualResetEventSlim();
var release = new ManualResetEventSlim();
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "authoritative")
{
entered.Set();
release.Wait(Wait);
}
});
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:reconcile");
await database.PutPredictedAsync(key, Note("predicted"));
var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text)));
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
var a = Task.Run(() => database.ApplyPutAsync(message));
Assert.That(entered.Wait(Wait), Is.True);
cache.Remove(key);
release.Set();
await a;
var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray();
Assert.Multiple(() =>
{
Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1),
"the removal is published as a removal: " + string.Join(", ", published));
Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text),
Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document");
Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True,
"no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document))));
});
}
// A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not
// hold (B removed it locally, a supported write) must still keep B's log contiguous for C.
[Test]
public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey()
{
const string shardId = "chain";
var schemaId = new CultCache().Registry.GetRequired<NetworkSchemaNote>().SchemaId;
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])]
});
var p = Make(new CultCache(), true);
var bCache = new CultCache();
var b = Make(bCache, false);
var c = Make(new CultCache(), false);
async Task Pull(CultNetDatabase from, CultNetDatabase to)
{
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = shardId,
ShardEpoch = 2,
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray()
});
}
var k = new CultRecordKey("soul:chain:k");
await p.PutAsync(k, Note("x"));
await Pull(p, b);
await Pull(b, c);
bCache.Remove(k);
await p.DeleteAsync<NetworkSchemaNote>(k);
await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y"));
await Pull(p, b);
Assert.Multiple(() =>
{
Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }),
"B's log holds every primary entry it applied");
Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap");
});
}
// A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction.
[Test]
public async Task S4_APredictionMadeInReactionToAReconcileSurvives()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:repredict");
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
var repredicted = false;
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
{
kinds.Enqueue(change.Kind);
if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted)
{
repredicted = true;
database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult();
}
});
CultNetDocumentPutRawMessage Put(string text)
{
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
return message;
}
await database.PutPredictedAsync(key, Note("predicted"));
await database.ApplyPutAsync(Put("auth1"));
await database.ApplyPutAsync(Put("auth2"));
Assert.That(kinds, Is.EqualTo(new[]
{
CultNetDatabaseChangeKind.Predicted,
CultNetDatabaseChangeKind.Reconciled,
CultNetDatabaseChangeKind.Predicted,
CultNetDatabaseChangeKind.Reconciled
}));
}
// Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A
// refused prediction leaves nothing behind that a later write of the same instance would wear.
[Test]
public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<CultNetDocumentRegistryVariantTests.VariantWireFixture>(registry));
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
DocumentRegistry = documents,
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var baseKey = new CultRecordKey("soul-refuse-base");
var variantKey = new CultRecordKey("soul-refuse-variant");
await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 },
new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<CultNetDocumentRegistryVariantTests.VariantWireFixture>(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v")
});
var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 };
Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException);
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
using var subscriber = database.Watch<CultNetDocumentRegistryVariantTests.VariantWireFixture>().Subscribe(change => kinds.Enqueue(change.Kind));
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
await cache.UpsertAsync(instance, new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(new CultRecordKey("soul-refuse-other")));
Assert.Multiple(() =>
{
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1));
});
}
// Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a
// durable log store can observe even though the disposed Subject hides publication.
[Test]
public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog()
{
var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N"));
var store = new CultNetFileShardMutationLogStore(root);
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store });
var shard = database.Shards[0].ShardId;
await cache.UpsertAsync(Note("before"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:before")));
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
database.Dispose();
try { await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:after"))); }
catch { /* a throwing handler would also be a failure below */ }
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
System.IO.Directory.Delete(root, true);
}
// The log entry exists before its change is published: a subscriber that reads the log from its handler finds it.
[Test]
public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("soul:log-first");
var loggedWhenPublished = new ConcurrentQueue<bool>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value)));
await database.PutAsync(key, Note("one"));
cache.Remove(key);
Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true }));
}
// An observer that mutates the admitted instance in place and upserts it again writes a second change of the same
// instance. It is published as its own update and never wears the context of the door's change, even when it
// reaches the database first.
[Test]
public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext()
{
var cache = new CultCache();
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "authoritative")
{
change.Document.Text = "normalized";
cache.UpsertAsync(change.Document, new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
}
});
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:in-place");
var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text)));
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
await database.PutPredictedAsync(key, Note("predicted"));
await database.ApplyPutAsync(message);
Assert.That(seen, Is.EqualTo(new (CultNetDatabaseChangeKind, string?)[]
{
(CultNetDatabaseChangeKind.Predicted, null),
(CultNetDatabaseChangeKind.Reconciled, "predicted"),
(CultNetDatabaseChangeKind.Updated, "normalized")
}));
}
}
}

View file

@ -0,0 +1,429 @@
#nullable enable
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests;
namespace GameCult.Networking.Tests
{
// The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the
// mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every
// other change in the hold publishes from the cache.
public sealed class CultNetPublishesCommitsTests
{
private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) =>
wire.Changes.Where(change => KeyOf(change) == key).ToArray();
private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey;
private sealed class Wire : IDisposable
{
private readonly CancellationTokenSource _cancellation = new();
private readonly Thread _thread;
private readonly RudpCultNetSchemaServer _server;
private readonly CultNetDatabaseSubscriptionServer _subscriptions;
private readonly ICultNetSchemaClient _client;
public ConcurrentQueue<CultNetDatabaseChangeRawMessage> Changes { get; } = new();
public int Snapshots;
public ConcurrentQueue<CultNetErrorMessage> Errors { get; } = new();
private readonly CultNetDatabase _database;
public Wire(CultNetDatabase database)
{
_database = database;
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
socket.ReceiveTimeout = 20;
_server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket });
_subscriptions = new CultNetDatabaseSubscriptionServer(_server, database);
_thread = new Thread(() =>
{
while (!_cancellation.IsCancellationRequested)
{
_ = _server.PollOnceAsync().GetAwaiter().GetResult();
Thread.Sleep(1);
}
}) { IsBackground = true };
_thread.Start();
_client = CultNetSchemaClients.CreateRudp("publish-client");
_client.OnCultNet<CultNetDatabaseChangeRawMessage>(Changes.Enqueue);
_client.OnCultNet<CultNetErrorMessage>(Errors.Enqueue);
_client.OnCultNet<CultNetSnapshotResponseRawMessage>(_ => Interlocked.Increment(ref Snapshots));
_client.Connect("127.0.0.1", _server.LocalEndPoint.Port);
}
public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes)
{
await Until(() => _client.Connected);
_client.SendCultNet(new CultNetDatabaseSubscribeMessage
{
MessageId = "subscribe",
SubscriptionId = "all",
SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(),
IncludeSnapshot = includeSnapshot
});
await Until(() => Snapshots > 0 || !Errors.IsEmpty);
Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused");
}
public void Dispose()
{
_cancellation.Cancel();
_thread.Join();
_client.Dispose();
_subscriptions.Dispose();
_server.Dispose();
}
}
private static async Task Until(Func<bool> condition)
{
var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5);
while (!condition())
{
if (DateTimeOffset.UtcNow > deadline)
Assert.Fail("Condition was not satisfied before timeout.");
await Task.Delay(5);
}
}
// One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance
// to arrive twice.
private static async Task SettleAsync(CultNetDatabase database, Wire wire)
{
var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N");
await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" });
await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel));
}
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
[Test]
public async Task ACacheCommitAndUpsertReachASubscriberOnce()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:commit");
cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle<NetworkSchemaNote>(key)));
await Until(() => Of(wire, key.Value).Length >= 1);
cache.Commit(batch => batch.Remove(key));
await Until(() => Of(wire, key.Value).Length >= 2);
await cache.UpsertAsync(Note("upserted"), new CultRecordHandle<NetworkSchemaNote>(key));
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3));
}
[Test]
public async Task ADatabasePutAndDeleteStillPublishExactlyOnce()
{
var database = new CultNetDatabase(new CultCache());
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:put");
await database.PutAsync(key, Note("put"));
await database.DeleteAsync<NetworkSchemaNote>(key);
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
}
[Test]
public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:replicated");
var message = database.Documents.CreateRawDocumentPutMessage(
"put-replicated", new CultRecordHandle<NetworkSchemaNote>(key), Note("remote"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
await database.ApplyPutAsync(message);
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1));
}
[Test]
public async Task ALoadRemovalCarriesItsKey()
{
var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc");
try
{
var key = new CultRecordKey("tests:publish:loaded");
var writer = new CultCache();
writer.AddBackingStore(new SingleFileMessagePackBackingStore(path));
await writer.UpsertAsync(Note("there"), new CultRecordHandle<NetworkSchemaNote>(key));
writer.FlushAllBackingStores();
var cache = new CultCache();
cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
var database = new CultNetDatabase(cache);
Assert.That(cache.Get<NetworkSchemaNote>(key), Is.Not.Null);
using var wire = new Wire(database);
await wire.SubscribeAsync(true, typeof(NetworkSchemaNote));
writer.Remove(key);
writer.FlushAllBackingStores();
await cache.PullAllBackingStoresAsync();
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" }));
}
finally
{
if (File.Exists(path)) File.Delete(path);
if (File.Exists(path + ".lock")) File.Delete(path + ".lock");
}
}
[Test]
public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big"),
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Power), 9)
});
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(VariantWireFixture));
var errorsBefore = wire.Errors.Count;
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
await Until(() => wire.Errors.Count > errorsBefore);
await Until(() => Of(wire, baseKey.Value).Length >= 1);
Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value),
"the re-resolved dependent is published, then refused (Q6) by key");
Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1));
Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record");
}
// An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own.
[Test]
public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:normalized");
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "raw")
cache.UpsertAsync(Note("normalized"), new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
});
await database.PutAsync(key, Note("raw"));
await Until(() => Of(wire, key.Value).Length >= 2);
await SettleAsync(database, wire);
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("normalized"));
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
}
private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
return (cache, documents);
}
private static System.Collections.Generic.List<string> Record(CultNetDatabase database, Func<object, string> describe)
{
var seen = new System.Collections.Generic.List<string>();
database.WatchAllChanges().Subscribe(change => seen.Add(describe(change)));
return seen;
}
private static string KeyOfChange(object change) => ((dynamic)change).Key.Value;
[Test]
public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder()
{
var (cache, documents) = VariantCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
});
var published = Record(database, KeyOfChange);
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value),
Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
}
private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) =>
new(cache, new CultNetDatabaseOptions
{
DocumentRegistry = documents,
Shards =
[
new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId])
]
});
[Test]
public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally()
{
var (sourceCache, sourceDocuments) = VariantCache();
var (replicaCache, replicaDocuments) = VariantCache();
var schemaId = sourceCache.Registry.GetRequired<VariantWireFixture>().SchemaId;
var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions
{
DocumentRegistry = sourceDocuments,
Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])]
});
var replica = Replica(replicaCache, replicaDocuments, schemaId);
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[]
{
replicaCache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
});
await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = "replica-shard",
ShardEpoch = 3,
Entries = source.GetMutationLogMessages("replica-shard").ToArray()
});
Assert.That(replicaCache.Get<VariantWireFixture>(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved");
Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }),
"the replica log holds the primary's entry and mints none of its own");
}
[Test]
public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged()
{
var cache = new CultCache();
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
var replica = Replica(cache, null, schemaId);
var published = Record(replica, KeyOfChange);
await cache.UpsertAsync(Note("local"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:local")));
Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" }));
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
}
[Test]
public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
const int writers = 4, perWriter = 5000;
Parallel.For(0, writers, writer =>
{
for (var i = 0; i < perWriter; i++)
cache.UpsertAsync(Note("n"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey($"tests:publish:c:{writer}:{i}")))
.GetAwaiter().GetResult();
});
var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray();
Assert.That(sequences, Has.Length.EqualTo(writers * perWriter));
Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter));
}
[Test]
public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("tests:publish:after-delete");
await database.PutAsync(key, Note("one"));
await database.DeleteAsync<NetworkSchemaNote>(key);
var published = Record(database, KeyOfChange);
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(key));
Assert.That(published, Is.EqualTo(new[] { key.Value }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind),
Is.EqualTo(new[]
{
CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added
}));
}
[Test]
public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing()
{
var cache = new CultCache();
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
var replica = Replica(cache, null, schemaId);
var stale = new CultRecordKey("tests:publish:stale");
var fresh = new CultRecordKey("tests:publish:fresh");
await cache.UpsertAsync(Note("stale"), new CultRecordHandle<NetworkSchemaNote>(stale));
var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value);
var put = replica.Documents.CreateRawDocumentPutMessage(
"snapshot-fresh", new CultRecordHandle<NetworkSchemaNote>(fresh), Note("fresh"));
await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage
{
ShardId = "replica-shard",
ShardEpoch = 3,
Documents = [put.Document!],
ShardLogSequence = 5
});
Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value }));
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
}
[Test]
public async Task ADisposedDatabaseNoLongerObservesTheCache()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var published = Record(database, KeyOfChange);
database.Dispose();
await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:disposed")));
Assert.That(published, Is.Empty);
}
}
}

View file

@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests
Assert.That(cache.Get(batched), Is.Null);
}), Is.True);
Assert.That(published, Is.EqualTo(new[] { put }));
Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it");
Assert.That(cache.Get<NetworkSchemaNote>(batched)?.Text, Is.EqualTo("batched"));
var reopened = new CultCache();
reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path));
@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests
var cache = new CultCache();
var delivering = new ManualResetEventSlim();
cache.OnUpdate += (_, _) => delivering.Set();
// Subscribed before the database, so it runs before the database's own observer of the same change.
using var loadDelivering = cache.Watch<object>().Subscribe(_ => delivering.Set());
var database = new CultNetDatabase(cache);
var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions
{