Merge publication Cut 4: CultNetDatabase rebuilt on the cache journal

Kind, prediction and log are decided in the journal under the cache gate;
the reorder buffer, drainer and prediction gate are gone. Q-P5 A: one
typed CultNetShardLogException per journal call, after publication. The
four old log bugs are fixed. No shard owns a schema means no write on any
path (typed refusal, wire code unowned_schema) and no row served; replicas
relay skipped entries unchanged. Four Soul passes; follow-ups recorded in
docs/cultcache-publication-cut.md (crash-safe log store is a hard
prerequisite before any live daemon enables durable shard logs).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-30 08:13:38 +02:00
commit 5b42eb0250
24 changed files with 3732 additions and 349 deletions

View file

@ -104,6 +104,16 @@ Not implemented yet:
- cross-runtime rollback/resimulation helpers for simulation frames
- declared CRDT merge policies
## Error Codes
`cultnet.error.v0` carries an optional machine-readable `code`. The set of codes is a closed enum
(`cultnet.error.schema.json`): Rust and TypeScript refuse a message whose code they do not know, they do
not guess. Adding a code is therefore a coordinated change across every runtime, made together: the schema (both
copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in
`packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are
`selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported` and
`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns).
## Live Invariants
- CultCache owns document schema, identity, local indexes, and local diffing.

View file

@ -26,9 +26,9 @@
}
},
"code": {
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.",
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).",
"type": ["string", "null"],
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"]
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"]
},
"details": {
"description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.",
@ -36,8 +36,8 @@
"additionalProperties": false,
"required": ["field", "value", "asOf", "current"],
"properties": {
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] },
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] },
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] },
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] },
"asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 },
"current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 }
}

View file

@ -26,9 +26,9 @@
}
},
"code": {
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.",
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).",
"type": ["string", "null"],
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"]
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"]
},
"details": {
"description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.",
@ -36,8 +36,8 @@
"additionalProperties": false,
"required": ["field", "value", "asOf", "current"],
"properties": {
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] },
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] },
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] },
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] },
"asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 },
"current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 }
}

View file

@ -250,8 +250,8 @@ impl CultNetShardDescriptor {
}
}
/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings
/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth
/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the strings
/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a
/// string it does not know refuses to decode rather than guess a meaning for it.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@ -261,6 +261,7 @@ pub enum CultNetErrorCode {
CursorInvalid,
ReferenceOutsideTarget,
VariantUnsupported,
UnownedSchema,
}
impl CultNetErrorCode {
@ -271,6 +272,7 @@ impl CultNetErrorCode {
Self::CursorInvalid => "cursor_invalid",
Self::ReferenceOutsideTarget => "reference_outside_target",
Self::VariantUnsupported => "variant_unsupported",
Self::UnownedSchema => "unowned_schema",
}
}
@ -281,6 +283,7 @@ impl CultNetErrorCode {
"cursor_invalid" => Ok(Self::CursorInvalid),
"reference_outside_target" => Ok(Self::ReferenceOutsideTarget),
"variant_unsupported" => Ok(Self::VariantUnsupported),
"unowned_schema" => Ok(Self::UnownedSchema),
other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")),
}
}

View file

@ -118,6 +118,46 @@ fn cases() -> Vec<Case> {
details: None,
},
},
Case {
name: "variant_unsupported_subscription_ended",
// C#: ForVariantUnsupported("variant unsupported here", "sub-1")
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0",
message: || CultNetMessage::Error {
error: "variant unsupported here".to_string(),
code: Some(CultNetErrorCode::VariantUnsupported),
details: Some(CultNetErrorDetails {
field: Some("subscriptionId".to_string()),
value: Some("sub-1".to_string()),
as_of: None,
current: None,
}),
},
},
Case {
name: "variant_unsupported_no_subscription",
// C#: ForVariantUnsupported("variant unsupported here") - no subscription ended, so Details stays null.
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0",
message: || CultNetMessage::Error {
error: "variant unsupported here".to_string(),
code: Some(CultNetErrorCode::VariantUnsupported),
details: None,
},
},
Case {
name: "unowned_schema",
// C#: ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")))
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0",
message: || CultNetMessage::Error {
error: "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.".to_string(),
code: Some(CultNetErrorCode::UnownedSchema),
details: Some(CultNetErrorDetails {
field: Some("schemaId".to_string()),
value: Some("some.schema.v1".to_string()),
as_of: None,
current: None,
}),
},
},
]
}

View file

@ -47,10 +47,20 @@ which route wrote it.
- Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a
per-cache, in-memory number assigned when the cache admits the change, under
its gate. It increases in admission order and is not persisted.
`OnUpdate(previous, current)` carries none. Streams CultNet derives from
`OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server,
database-backed Mesh handles) have no ordering guarantee and no stale
protection.
`OnUpdate(previous, current)` carries none. CultNet derives its streams
(`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed
Mesh handles) from `Watch` plus one journal (`AddJournal`), so a load, a
commit and a variant dependent's re-resolution all reach subscribers and the
mutation log. The journal runs under the gate in `Sequence` order, so the
shard log is written in cache order; the `Watch` observer publishes what the
journal decided. A database write door only hands the journal the wire message
or replicated entry for the change it admitted. A replica publishes but never
mints log sequences. A primary logs every change it admits, loads and pulls
included; a store attached after the database exists has its hydration logged
too. A primary that cannot log a change compacts through that change's
sequence and throws `CultNetShardLogException` to the writer after publication:
the commit stands, and a replica behind it resynchronizes from a snapshot. The
streams carry no `Sequence` and have no stale protection.
- `GetWithSequence(key)` returns the document and the cache's current
`Sequence`, read together under the gate: every change with a `Sequence` at
or below it is reflected in the document.
@ -126,6 +136,10 @@ which route wrote it.
needs the cache or its gate: another thread's write, a lock a writer holds.
Its own leaf locks, taken only inside the journal, are the only locks that may
nest under the gate.
- A shard mutation log store (`ICultNetShardMutationLogStore`) runs inside the
CultNet journal, under the cache's gate: it must not call the cache or wait
for a thread that does. A store that writes its own cache throws through the
journal's reentry guard, and the admission stands.
- A thread holds one cache's gate at a time: entering a cache's hold inside
another cache's hold throws.
- A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate,

View file

@ -2124,7 +2124,15 @@ namespace GameCult.Caching
Remove(handle.Key);
}
// false: a condition failed; nothing was written, changed in memory, or published.
/// <summary>
/// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published.
/// </summary>
/// <remarks>
/// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer
/// receives the failure. One failure arrives as itself, several as an <see cref="AggregateException"/>. A writer that
/// wants a particular failure (a CultNetShardLogException, when a database's log refused the commit) catches that type,
/// or unwraps <see cref="AggregateException.InnerExceptions"/> and looks for it.
/// </remarks>
public bool Commit(Action<CultCacheBatch> stage)
{
return Land(stage, wait: true) == CultCommitOutcome.Committed;
@ -2427,7 +2435,8 @@ namespace GameCult.Caching
// run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not
// guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate,
// in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even
// if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown.
// if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their
// origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions.
// If the body threw, its exception wins.
internal T Held<T>(Func<T> body)
{

View file

@ -3379,7 +3379,8 @@ namespace GameCult.Mesh
if (document != null)
return document;
var untyped = database.Cache.Get(key);
// Through the database, which serves only what its shards own; the cache behind it may hold more.
var untyped = await database.GetAsync<object>(key).ConfigureAwait(false);
if (untyped != null && IsSameCultDocumentSchema<TDocument>(untyped.GetType()))
return ConvertUntypedDocument<TDocument>(untyped);

File diff suppressed because it is too large Load diff

View file

@ -112,10 +112,13 @@ namespace GameCult.Networking
request);
}
// A database serves only what its shards own.
return _database.Documents.CreateRawSnapshotResponse(
_database.Cache,
string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId,
request);
request,
options: null,
rowFilter: _database.Owns);
}
/// <summary>
@ -135,6 +138,9 @@ namespace GameCult.Networking
// R-Q: asOf is one shard-log watermark, so a selection whose matched rows span more than
// one shard's log is refused rather than answered against a watermark that is not
// exact for all of them.
// A row whose schema no shard owns is not part of what this database serves: it is left out, not an error.
options: null,
rowFilter: _database.Owns,
shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId,
cursorKey: _database.CursorKey,
// S-9: once the matched rows resolve to one shard, asOf is that shard's own watermark,
@ -312,6 +318,16 @@ namespace GameCult.Networking
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (CultNetUnownedSchemaException ex)
{
_server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}");
peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex));
}
catch (Exception ex) when (CommittedWithLogFailure(ex))
{
// The write committed and was published; only its shard log failed, so the peer is not told it was refused.
_server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}");
}
catch (Exception ex)
{
_server.Logger.LogError($"CultNet raw put failed: {ex.Message}");
@ -332,6 +348,15 @@ namespace GameCult.Networking
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (CultNetUnownedSchemaException ex)
{
_server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}");
peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex));
}
catch (Exception ex) when (CommittedWithLogFailure(ex))
{
_server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}");
}
catch (Exception ex)
{
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
@ -604,6 +629,12 @@ namespace GameCult.Networking
: messageId;
}
// A CultNetShardLogException means the write committed and was published (the writer hears of a failed log afterwards);
// the cache wraps it in an AggregateException when something else failed in the same hold.
private static bool CommittedWithLogFailure(Exception exception) =>
exception is CultNetShardLogException ||
exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException);
private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception)
{
return new CultNetErrorMessage

View file

@ -421,7 +421,8 @@ namespace GameCult.Networking
documentSelection,
ordinalOf: (schemaId, key) => _database.LastWriteSequence(schemaId, key) ?? 0,
asOf: _database.CurrentAsOf(),
options: null);
options: null,
rowFilter: _database.Owns);
// The snapshot is one row per record key by construction, so the duplicate-key throw this
// used to guard can no longer fire (docs/cultnet-selection-cut.md, S2-2); deleted rather
// than tested around.

View file

@ -280,7 +280,16 @@ namespace GameCult.Networking
CultCache cache,
string messageId,
CultNetSnapshotRequestMessage? filter = null,
CultNetDocumentMessageOptions? options = null)
CultNetDocumentMessageOptions? options = null) =>
CreateRawSnapshotResponse(cache, messageId, filter, options, rowFilter: null);
// rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own).
internal CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse(
CultCache cache,
string messageId,
CultNetSnapshotRequestMessage? filter,
CultNetDocumentMessageOptions? options,
Func<CultDocumentDescriptor, CultRecordKey, bool>? rowFilter)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds);
@ -323,7 +332,7 @@ namespace GameCult.Networking
ordinalOf = static (_, _) => 0;
}
var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options);
var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options, rowFilter);
return new CultNetSnapshotResponseRawMessage
{
@ -348,13 +357,27 @@ namespace GameCult.Networking
CultNetDocumentMessageOptions? options = null,
Func<string, CultRecordKey, string>? shardIdOf = null,
CultNetSelectionCursorKey? cursorKey = null,
Func<string, ulong>? asOfForShard = null)
Func<string, ulong>? asOfForShard = null) =>
CreateSelectionResponse(cache, messageId, selection, ordinalOf, asOf, options, shardIdOf, cursorKey, asOfForShard, rowFilter: null);
// rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own).
internal CultNetSnapshotResponseRawV1Message CreateSelectionResponse(
CultCache cache,
string messageId,
CultNetSelection selection,
Func<string, CultRecordKey, long> ordinalOf,
ulong asOf,
CultNetDocumentMessageOptions? options,
Func<string, CultRecordKey, string>? shardIdOf,
CultNetSelectionCursorKey? cursorKey,
Func<string, ulong>? asOfForShard,
Func<CultDocumentDescriptor, CultRecordKey, bool>? rowFilter)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
if (selection == null) throw new ArgumentNullException(nameof(selection));
if (ordinalOf == null) throw new ArgumentNullException(nameof(ordinalOf));
var page = SelectPage(cache, selection, ordinalOf, asOf, options, shardIdOf: shardIdOf, cursorKey: cursorKey, asOfForShard: asOfForShard);
var page = SelectPage(cache, selection, ordinalOf, asOf, options, rowFilter, shardIdOf, cursorKey, asOfForShard);
return new CultNetSnapshotResponseRawV1Message
{
MessageId = RequireNonEmpty(messageId, nameof(messageId)),

View file

@ -511,7 +511,7 @@ namespace GameCult.Networking
[Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; }
/// <summary>
/// Gets or sets the machine-readable refusal code (R-N), e.g. <c>selection_invalid</c>,
/// <c>cursor_stale</c>, <c>cursor_invalid</c>, <c>reference_outside_target</c>. Additive to
/// <c>cursor_stale</c>, <c>cursor_invalid</c>, <c>reference_outside_target</c>, <c>unowned_schema</c>. Additive to
/// <c>cultnet.error.v0</c>; a v0 peer that does not know this field ignores it.
/// </summary>
[Key("code")] public string? Code { get; set; }
@ -532,6 +532,17 @@ namespace GameCult.Networking
Details = new CultNetErrorDetails { Field = ex.Field, Value = ex.Value }
};
/// <summary>
/// Builds the wire error for a write no shard owns: code <c>unowned_schema</c>, details <c>{ field: "schemaId", value }</c>.
/// </summary>
public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) =>
new CultNetErrorMessage
{
Error = $"unowned_schema: {ex.Message}",
Code = "unowned_schema",
Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId }
};
/// <summary>
/// Builds the wire error for a cursor refusal (R-N): code <c>cursor_stale</c> with details
/// <c>{ asOf, current }</c>, or code <c>cursor_invalid</c> with no details.

View file

@ -10,6 +10,11 @@ namespace GameCult.Networking
/// <summary>
/// Stores accepted shard-log entries in their replica catch-up wire shape.
/// </summary>
/// <remarks>
/// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of
/// the cache and blocks its readers too, because reads take the same gate. A store must not call the cache, or wait for a thread that does. Appends must throw rather than
/// swallow a failure: the database burns the sequence and compacts past it, and tells the writer.
/// </remarks>
public interface ICultNetShardMutationLogStore
{
/// <summary>

View file

@ -938,6 +938,32 @@ public sealed class CultMeshStreamingTests
await act.Should().ThrowAsync<NotSupportedException>();
}
// A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type
// out of the cache behind it.
[TestCase("tests.mesh_note.v1", true)]
[TestCase("tests.unrelated.v1", false)]
public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served)
{
var cache = new CultCache();
var key = new CultRecordKey("note:one");
await cache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cached" }, new CultRecordHandle<MeshNoteDocument>(key));
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "reader",
Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) }
});
var handle = CultMesh.Document<MeshNoteAliasDocument>(database, key, CultMesh.Verse("starbridge", "reader"));
if (served)
{
(await handle.LatestAsync()).Text.Should().Be("cached");
}
else
{
await FluentActions.Awaiting(() => handle.LatestAsync()).Should().ThrowAsync<KeyNotFoundException>();
}
}
[Test]
public async Task DocumentHandle_SubmitsPredictionsThroughCultNetDatabase()
{

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,351 @@
#nullable enable
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
public sealed class CultNetPublicationOrderTests
{
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text;
// A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed
// (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile.
[Test]
public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder()
{
var cache = new CultCache();
var entered = new ManualResetEventSlim();
var release = new ManualResetEventSlim();
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "A")
{
entered.Set();
release.Wait(Wait);
}
});
var database = new CultNetDatabase(cache);
var seen = new List<string?>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); });
var key = new CultRecordKey("soul:order");
var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult());
Assert.That(entered.Wait(Wait), Is.True);
await database.PutAsync(key, Note("B"));
release.Set();
await a;
var shard = database.Shards[0].ShardId;
var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last();
// A replica built from the log.
var replicaCache = new CultCache();
var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)]
});
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = shard,
ShardEpoch = database.Shards[0].Epoch,
Entries = database.GetMutationLogMessages(shard).ToArray()
});
Assert.Multiple(() =>
{
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B");
Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write");
Assert.That(replicaCache.Get<NetworkSchemaNote>(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary");
// Delivery is not ordered across threads (order is data: a wire peer drops the stale change by Sequence, Cut 6),
// so a subscriber sees both changes, once each, in either order.
Assert.That(seen, Is.EquivalentTo(new[] { "A", "B" }), "each change is published exactly once");
});
}
// Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache.
[Test]
public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing()
{
var divergent = 0;
const int rounds = 300;
for (var round = 0; round < rounds; round++)
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("soul:hammer");
using var start = new Barrier(2);
Parallel.For(0, 2, writer =>
{
start.SignalAndWait();
for (var i = 0; i < 40; i++)
cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult();
});
var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last();
if (!ReferenceEquals(last.Document, cache.Get<NetworkSchemaNote>(key)))
divergent++;
}
TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}");
Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache");
}
// A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in
// the window before the door's own change is handled, consumes the door's context.
[Test]
public async Task S2_ARemovalNeverWearsAReconcilingPutsContext()
{
var cache = new CultCache();
var entered = new ManualResetEventSlim();
var release = new ManualResetEventSlim();
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "authoritative")
{
entered.Set();
release.Wait(Wait);
}
});
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:reconcile");
await database.PutPredictedAsync(key, Note("predicted"));
var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text)));
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
var a = Task.Run(() => database.ApplyPutAsync(message));
Assert.That(entered.Wait(Wait), Is.True);
cache.Remove(key);
release.Set();
await a;
var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray();
Assert.Multiple(() =>
{
Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1),
"the removal is published as a removal: " + string.Join(", ", published));
Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text),
Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document");
Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True,
"no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document))));
});
}
// A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not
// hold (B removed it locally, a supported write) must still keep B's log contiguous for C.
[Test]
public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey()
{
const string shardId = "chain";
var schemaId = new CultCache().Registry.GetRequired<NetworkSchemaNote>().SchemaId;
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])]
});
var p = Make(new CultCache(), true);
var bCache = new CultCache();
var b = Make(bCache, false);
var c = Make(new CultCache(), false);
async Task Pull(CultNetDatabase from, CultNetDatabase to)
{
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = shardId,
ShardEpoch = 2,
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray()
});
}
var k = new CultRecordKey("soul:chain:k");
await p.PutAsync(k, Note("x"));
await Pull(p, b);
await Pull(b, c);
bCache.Remove(k);
await p.DeleteAsync<NetworkSchemaNote>(k);
await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y"));
await Pull(p, b);
Assert.Multiple(() =>
{
Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }),
"B's log holds every primary entry it applied");
Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap");
});
}
// A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction.
[Test]
public async Task S4_APredictionMadeInReactionToAReconcileSurvives()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:repredict");
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
var repredicted = false;
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
{
kinds.Enqueue(change.Kind);
if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted)
{
repredicted = true;
database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult();
}
});
CultNetDocumentPutRawMessage Put(string text)
{
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
return message;
}
await database.PutPredictedAsync(key, Note("predicted"));
await database.ApplyPutAsync(Put("auth1"));
await database.ApplyPutAsync(Put("auth2"));
Assert.That(kinds, Is.EqualTo(new[]
{
CultNetDatabaseChangeKind.Predicted,
CultNetDatabaseChangeKind.Reconciled,
CultNetDatabaseChangeKind.Predicted,
CultNetDatabaseChangeKind.Reconciled
}));
}
// Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A
// refused prediction leaves nothing behind that a later write of the same instance would wear.
[Test]
public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<CultNetDocumentRegistryVariantTests.VariantWireFixture>(registry));
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
DocumentRegistry = documents,
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var baseKey = new CultRecordKey("soul-refuse-base");
var variantKey = new CultRecordKey("soul-refuse-variant");
await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 },
new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<CultNetDocumentRegistryVariantTests.VariantWireFixture>(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v")
});
var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 };
Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException);
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
using var subscriber = database.Watch<CultNetDocumentRegistryVariantTests.VariantWireFixture>().Subscribe(change => kinds.Enqueue(change.Kind));
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
await cache.UpsertAsync(instance, new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(new CultRecordKey("soul-refuse-other")));
Assert.Multiple(() =>
{
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1));
});
}
// Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a
// durable log store can observe even though the disposed Subject hides publication.
[Test]
public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog()
{
var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N"));
var store = new CultNetFileShardMutationLogStore(root);
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store });
var shard = database.Shards[0].ShardId;
await cache.UpsertAsync(Note("before"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:before")));
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
database.Dispose();
try { await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:after"))); }
catch { /* a throwing handler would also be a failure below */ }
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
System.IO.Directory.Delete(root, true);
}
// The log entry exists before its change is published: a subscriber that reads the log from its handler finds it.
[Test]
public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("soul:log-first");
var loggedWhenPublished = new ConcurrentQueue<bool>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value)));
await database.PutAsync(key, Note("one"));
cache.Remove(key);
Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true }));
}
// An observer that mutates the admitted instance in place and upserts it again writes a second change of the same
// instance. It is published as its own update and never wears the context of the door's change, even when it
// reaches the database first.
[Test]
public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext()
{
var cache = new CultCache();
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "authoritative")
{
change.Document.Text = "normalized";
cache.UpsertAsync(change.Document, new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
}
});
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul:in-place");
var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>();
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text)));
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
await database.PutPredictedAsync(key, Note("predicted"));
await database.ApplyPutAsync(message);
// The observer's nested write is delivered depth-first, inside the delivery of the change that caused it, so
// the database may publish the two in either order; what is pinned is that each carries its own context.
Assert.That(seen, Is.EquivalentTo(new (CultNetDatabaseChangeKind, string?)[]
{
(CultNetDatabaseChangeKind.Predicted, null),
(CultNetDatabaseChangeKind.Reconciled, "predicted"),
(CultNetDatabaseChangeKind.Updated, "normalized")
}));
}
}
}

View file

@ -0,0 +1,429 @@
#nullable enable
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests;
namespace GameCult.Networking.Tests
{
// The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the
// mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every
// other change in the hold publishes from the cache.
public sealed class CultNetPublishesCommitsTests
{
private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) =>
wire.Changes.Where(change => KeyOf(change) == key).ToArray();
private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey;
private sealed class Wire : IDisposable
{
private readonly CancellationTokenSource _cancellation = new();
private readonly Thread _thread;
private readonly RudpCultNetSchemaServer _server;
private readonly CultNetDatabaseSubscriptionServer _subscriptions;
private readonly ICultNetSchemaClient _client;
public ConcurrentQueue<CultNetDatabaseChangeRawMessage> Changes { get; } = new();
public int Snapshots;
public ConcurrentQueue<CultNetErrorMessage> Errors { get; } = new();
private readonly CultNetDatabase _database;
public Wire(CultNetDatabase database)
{
_database = database;
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
socket.ReceiveTimeout = 20;
_server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket });
_subscriptions = new CultNetDatabaseSubscriptionServer(_server, database);
_thread = new Thread(() =>
{
while (!_cancellation.IsCancellationRequested)
{
_ = _server.PollOnceAsync().GetAwaiter().GetResult();
Thread.Sleep(1);
}
}) { IsBackground = true };
_thread.Start();
_client = CultNetSchemaClients.CreateRudp("publish-client");
_client.OnCultNet<CultNetDatabaseChangeRawMessage>(Changes.Enqueue);
_client.OnCultNet<CultNetErrorMessage>(Errors.Enqueue);
_client.OnCultNet<CultNetSnapshotResponseRawMessage>(_ => Interlocked.Increment(ref Snapshots));
_client.Connect("127.0.0.1", _server.LocalEndPoint.Port);
}
public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes)
{
await Until(() => _client.Connected);
_client.SendCultNet(new CultNetDatabaseSubscribeMessage
{
MessageId = "subscribe",
SubscriptionId = "all",
SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(),
IncludeSnapshot = includeSnapshot
});
await Until(() => Snapshots > 0 || !Errors.IsEmpty);
Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused");
}
public void Dispose()
{
_cancellation.Cancel();
_thread.Join();
_client.Dispose();
_subscriptions.Dispose();
_server.Dispose();
}
}
private static async Task Until(Func<bool> condition)
{
var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5);
while (!condition())
{
if (DateTimeOffset.UtcNow > deadline)
Assert.Fail("Condition was not satisfied before timeout.");
await Task.Delay(5);
}
}
// One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance
// to arrive twice.
private static async Task SettleAsync(CultNetDatabase database, Wire wire)
{
var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N");
await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" });
await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel));
}
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
[Test]
public async Task ACacheCommitAndUpsertReachASubscriberOnce()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:commit");
cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle<NetworkSchemaNote>(key)));
await Until(() => Of(wire, key.Value).Length >= 1);
cache.Commit(batch => batch.Remove(key));
await Until(() => Of(wire, key.Value).Length >= 2);
await cache.UpsertAsync(Note("upserted"), new CultRecordHandle<NetworkSchemaNote>(key));
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3));
}
[Test]
public async Task ADatabasePutAndDeleteStillPublishExactlyOnce()
{
var database = new CultNetDatabase(new CultCache());
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:put");
await database.PutAsync(key, Note("put"));
await database.DeleteAsync<NetworkSchemaNote>(key);
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
}
[Test]
public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:replicated");
var message = database.Documents.CreateRawDocumentPutMessage(
"put-replicated", new CultRecordHandle<NetworkSchemaNote>(key), Note("remote"));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
await database.ApplyPutAsync(message);
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1));
}
[Test]
public async Task ALoadRemovalCarriesItsKey()
{
var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc");
try
{
var key = new CultRecordKey("tests:publish:loaded");
var writer = new CultCache();
writer.AddBackingStore(new SingleFileMessagePackBackingStore(path));
await writer.UpsertAsync(Note("there"), new CultRecordHandle<NetworkSchemaNote>(key));
writer.FlushAllBackingStores();
var cache = new CultCache();
cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
var database = new CultNetDatabase(cache);
Assert.That(cache.Get<NetworkSchemaNote>(key), Is.Not.Null);
using var wire = new Wire(database);
await wire.SubscribeAsync(true, typeof(NetworkSchemaNote));
writer.Remove(key);
writer.FlushAllBackingStores();
await cache.PullAllBackingStoresAsync();
await SettleAsync(database, wire);
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" }));
}
finally
{
if (File.Exists(path)) File.Delete(path);
if (File.Exists(path + ".lock")) File.Delete(path + ".lock");
}
}
[Test]
public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big"),
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Power), 9)
});
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(VariantWireFixture));
var errorsBefore = wire.Errors.Count;
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
await Until(() => wire.Errors.Count > errorsBefore);
await Until(() => Of(wire, baseKey.Value).Length >= 1);
Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value),
"the re-resolved dependent is published, then refused (Q6) by key");
Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1));
Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record");
}
// An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own.
[Test]
public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
using var wire = new Wire(database);
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
var key = new CultRecordKey("tests:publish:normalized");
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "raw")
cache.UpsertAsync(Note("normalized"), new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
});
await database.PutAsync(key, Note("raw"));
await Until(() => Of(wire, key.Value).Length >= 2);
await SettleAsync(database, wire);
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("normalized"));
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
}
private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache()
{
var registry = CultDocumentRegistry.Shared;
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
var documents = new CultNetDocumentRegistry(registry)
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
return (cache, documents);
}
private static System.Collections.Generic.List<string> Record(CultNetDatabase database, Func<object, string> describe)
{
var seen = new System.Collections.Generic.List<string>();
database.WatchAllChanges().Subscribe(change => seen.Add(describe(change)));
return seen;
}
private static string KeyOfChange(object change) => ((dynamic)change).Key.Value;
[Test]
public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder()
{
var (cache, documents) = VariantCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
{
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
});
var published = Record(database, KeyOfChange);
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value),
Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
}
private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) =>
new(cache, new CultNetDatabaseOptions
{
DocumentRegistry = documents,
Shards =
[
new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId])
]
});
[Test]
public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally()
{
var (sourceCache, sourceDocuments) = VariantCache();
var (replicaCache, replicaDocuments) = VariantCache();
var schemaId = sourceCache.Registry.GetRequired<VariantWireFixture>().SchemaId;
var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions
{
DocumentRegistry = sourceDocuments,
Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])]
});
var replica = Replica(replicaCache, replicaDocuments, schemaId);
var baseKey = new CultRecordKey("publish-variant-base");
var variantKey = new CultRecordKey("publish-variant-big");
await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[]
{
replicaCache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
});
await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = "replica-shard",
ShardEpoch = 3,
Entries = source.GetMutationLogMessages("replica-shard").ToArray()
});
Assert.That(replicaCache.Get<VariantWireFixture>(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved");
Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }),
"the replica log holds the primary's entry and mints none of its own");
}
[Test]
public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged()
{
var cache = new CultCache();
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
var replica = Replica(cache, null, schemaId);
var published = Record(replica, KeyOfChange);
await cache.UpsertAsync(Note("local"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:local")));
Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" }));
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
}
[Test]
public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
const int writers = 4, perWriter = 5000;
Parallel.For(0, writers, writer =>
{
for (var i = 0; i < perWriter; i++)
cache.UpsertAsync(Note("n"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey($"tests:publish:c:{writer}:{i}")))
.GetAwaiter().GetResult();
});
var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray();
Assert.That(sequences, Has.Length.EqualTo(writers * perWriter));
Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter));
}
[Test]
public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var key = new CultRecordKey("tests:publish:after-delete");
await database.PutAsync(key, Note("one"));
await database.DeleteAsync<NetworkSchemaNote>(key);
var published = Record(database, KeyOfChange);
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(key));
Assert.That(published, Is.EqualTo(new[] { key.Value }));
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind),
Is.EqualTo(new[]
{
CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added
}));
}
[Test]
public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing()
{
var cache = new CultCache();
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
var replica = Replica(cache, null, schemaId);
var stale = new CultRecordKey("tests:publish:stale");
var fresh = new CultRecordKey("tests:publish:fresh");
await cache.UpsertAsync(Note("stale"), new CultRecordHandle<NetworkSchemaNote>(stale));
var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value);
var put = replica.Documents.CreateRawDocumentPutMessage(
"snapshot-fresh", new CultRecordHandle<NetworkSchemaNote>(fresh), Note("fresh"));
await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage
{
ShardId = "replica-shard",
ShardEpoch = 3,
Documents = [put.Document!],
ShardLogSequence = 5
});
Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value }));
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
}
[Test]
public async Task ADisposedDatabaseNoLongerObservesTheCache()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var published = Record(database, KeyOfChange);
database.Dispose();
await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:disposed")));
Assert.That(published, Is.Empty);
}
}
}

View file

@ -195,6 +195,17 @@ namespace GameCult.Networking.Tests
AssertErrorMatchesSchema(CultNetErrorMessage.ForReferenceOutsideTarget(ex));
}
[Test]
public void ErrorMessage_UnownedSchemaRealBytes_MatchesItsSchema()
{
var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")));
Assert.That(message.Code, Is.EqualTo("unowned_schema"));
Assert.That(message.Details!.Field, Is.EqualTo("schemaId"));
Assert.That(message.Details.Value, Is.EqualTo("some.schema.v1"));
AssertErrorMatchesSchema(message);
}
private static void AssertErrorMatchesSchema(CultNetErrorMessage message)
{
var validator = new MiniJsonSchemaValidator(SchemaDir());

View file

@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests
Assert.That(cache.Get(batched), Is.Null);
}), Is.True);
Assert.That(published, Is.EqualTo(new[] { put }));
Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it");
Assert.That(cache.Get<NetworkSchemaNote>(batched)?.Text, Is.EqualTo("batched"));
var reopened = new CultCache();
reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path));
@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests
var cache = new CultCache();
var delivering = new ManualResetEventSlim();
cache.OnUpdate += (_, _) => delivering.Set();
// Subscribed before the database, so it runs before the database's own observer of the same change.
using var loadDelivering = cache.Watch<object>().Subscribe(_ => delivering.Set());
var database = new CultNetDatabase(cache);
var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions
{
@ -4211,13 +4212,13 @@ namespace GameCult.Networking.Tests
Schema = "tests.networking_note.v1",
Text = "two"
});
// Outside the shard's key prefix: must reach the non-shard snapshot but never the
// Outside the shard's key prefix (written to the cache directly: the database refuses a write no shard owns): must reach the non-shard snapshot but never the
// shard-scoped one, so an unfiltered request still proves shard membership is enforced.
await database.PutAsync(new CultRecordKey("other-note:outside-shard"), new NetworkSchemaNote
await cache.UpsertAsync(new NetworkSchemaNote
{
Schema = "tests.networking_note.v1",
Text = "outside"
});
}, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("other-note:outside-shard")));
var nonShard = registry.CreateRawSnapshotResponse(cache, "snapshot-nonshard-unfiltered", filter: null);
var shardScoped = database.CreateShardSnapshotResponse(shard, "snapshot-shard-unfiltered", filter: null);

View file

@ -0,0 +1,304 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
// Soul pass on Cut 4 (scratch; not for merge). Category Finding asserts the invariant Soul expects and is expected to
// fail on 13072fc5; Pin asserts a promise that should hold; Measure reports numbers.
public sealed class SoulP4ProbeTests
{
private const string ShardId = "soul-shard";
private static readonly CultRecordKey One = new("soul:p4:one");
private static readonly CultRecordKey Two = new("soul:p4:two");
private static readonly CultRecordKey Three = new("soul:p4:three");
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
private static string SchemaId(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) =>
new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])],
MutationLogStore = store
});
private sealed class Store : ICultNetShardMutationLogStore
{
private readonly object _gate = new();
private readonly SortedDictionary<long, CultNetShardLogEntryMessage> _entries = new();
private long _compacted;
public Func<long, bool> RefuseAppend { get; set; } = _ => false;
public bool RefuseCompact { get; set; }
public int AppendDelayMs { get; set; }
public ManualResetEventSlim InAppend { get; } = new();
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null)
{
lock (_gate)
{
var e = _entries.Values.Where(x => x.Sequence > afterSequence);
return (limit.HasValue ? e.Take(limit.Value) : e).ToArray();
}
}
public void Append(string shardId, CultNetShardLogEntryMessage entry)
{
InAppend.Set();
if (AppendDelayMs > 0) Thread.Sleep(AppendDelayMs);
lock (_gate)
{
if (RefuseAppend(entry.Sequence)) throw new IOException($"refused {entry.Sequence}");
_entries[entry.Sequence] = entry;
}
}
public long GetCompactedThrough(string shardId) { lock (_gate) return _compacted; }
public void CompactThrough(string shardId, long sequence)
{
lock (_gate)
{
if (RefuseCompact) throw new IOException("refused compact");
_compacted = Math.Max(_compacted, sequence);
foreach (var k in _entries.Keys.Where(k => k <= sequence).ToArray()) _entries.Remove(k);
}
}
}
private sealed class Peer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable
{
private readonly Server _server;
private readonly CultNetDatabaseServer _dbServer;
private readonly CultNetDatabase _db;
public Peer(CultCache cache, CultNetDatabase db)
{
_db = db;
_server = new Server(cache, ServerSecurityOptions.Development());
_dbServer = new CultNetDatabaseServer(_server, db);
}
public CultNetShardLogResponseMessage Log(long after) => _dbServer.CreateShardLogResponse(new CultNetShardLogRequestMessage
{ MessageId = "pull", ShardId = ShardId, ShardEpoch = 1, AfterSequence = after });
public Task<CultNetShardLogResponseMessage> FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => Task.FromResult(Log(afterSequence));
public Task<CultNetSnapshotResponseRawMessage> FetchAsync(CultNetShardDescriptor shard) => Task.FromResult(_db.CreateShardSnapshotResponse(_db.Shards[0], "snap"));
public void Dispose() { _dbServer.Dispose(); _server.Dispose(); }
}
private static int StashCount(CultNetDatabase db)
{
var field = typeof(CultNetDatabase).GetField("_stash", BindingFlags.NonPublic | BindingFlags.Instance)!;
var stash = (System.Collections.ICollection)field.GetValue(db)!;
lock (typeof(CultNetDatabase).GetField("_logGate", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(db)!)
return stash.Count;
}
// F1: the recorded residual is "tail reuse". A hole in the middle of the log after a double refusal and a restart
// strands a replica behind it for good: the restarted primary has no floor, the store serves S+1 after S-1.
[Test, Category("SoulFinding")]
public async Task F1_MidLogHoleAfterDoubleRefusalAndRestartStrandsAReplica()
{
var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true };
var cache = new CultCache();
var primary = Db(cache, true, store);
var replicaCache = new CultCache();
var replica = Db(replicaCache, false, null);
await primary.PutAsync(One, Note("one"));
using (var p0 = new Peer(cache, primary))
using (var r0 = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = p0, SnapshotFetcher = p0 }))
await r0.PullOnceAsync(ShardId);
try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { }
store.RefuseCompact = false;
await primary.PutAsync(Three, Note("three"));
primary.Dispose();
var restarted = Db(cache, true, store);
using var peer = new Peer(cache, restarted);
using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer });
var errors = new List<string>();
for (var attempt = 0; attempt < 3; attempt++)
{
try { await replicator.PullOnceAsync(ShardId); }
catch (Exception e) { errors.Add(e.GetType().Name + ": " + e.Message); }
}
TestContext.Out.WriteLine($"store={string.Join(",", store.Read(ShardId).Select(e => e.Sequence))} floor={restarted.GetCompactedMutationLogSequence(ShardId)} errors=[{string.Join(" | ", errors)}]");
Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get<NetworkSchemaNote>(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" }));
}
// F2: Q-P5 A says the writer gets a typed CultNetShardLogException. A commit of two changes on a refusing store
// hands the writer an AggregateException; a writer's catch (CultNetShardLogException) misses it.
[Test, Category("SoulFinding")]
public void F2_ABatchOnARefusingStoreReachesTheWriterAsTheTypedException()
{
var store = new Store { RefuseAppend = _ => true };
var cache = new CultCache();
_ = Db(cache, true, store);
Exception? caught = null;
try
{
cache.Commit(batch =>
{
batch.Upsert(Note("one"), new CultRecordHandle<NetworkSchemaNote>(One));
batch.Upsert(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
});
}
catch (Exception e) { caught = e; }
TestContext.Out.WriteLine($"writer caught {caught?.GetType().FullName}");
Assert.That(caught, Is.InstanceOf<CultNetShardLogException>());
}
// F3: main publishes PutPredictedAsync as Predicted and records the prediction whatever instance it is given. The
// shape rule refuses a door whose instance is already the cached one, so a mutate-then-predict is published as
// Updated, is logged on a primary, and its reconciliation is published as Updated.
[Test, Category("SoulFinding")]
public async Task F3_APredictionOfTheCachedInstanceIsPublishedAsAPrediction()
{
var cache = new CultCache();
var db = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var shard = db.Shards[0];
var kinds = new List<CultNetDatabaseChangeKind>();
db.Watch<NetworkSchemaNote>().Subscribe(c => kinds.Add(c.Kind));
await db.PutPredictedAsync(One, Note("p1"));
var held = cache.Get<NetworkSchemaNote>(One)!;
held.Text = "p2";
await db.PutPredictedAsync(One, held);
var auth = db.Documents.CreateRawDocumentPutMessage("a", new CultRecordHandle<NetworkSchemaNote>(One), Note("auth"));
auth.ShardId = shard.ShardId;
auth.ShardEpoch = shard.Epoch;
await db.ApplyPutAsync(auth);
TestContext.Out.WriteLine($"kinds=[{string.Join(",", kinds)}] log=[{string.Join(",", db.GetMutationLog(shard.ShardId).Select(e => e.Kind))}]");
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled }));
Assert.That(db.GetMutationLog(shard.ShardId).Select(e => e.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated }));
}
// F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an
// unrelated key waits for the whole append.
[Test, Category("SoulFinding"), Ignore("Accepted follow-up F4: a log append runs under the cache gate, so readers wait; documented on ICultNetShardMutationLogStore.")]
public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend()
{
var store = new Store { AppendDelayMs = 400 };
var cache = new CultCache();
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
var db = Db(cache, true, store);
var writer = Task.Run(() => cache.UpsertAsync(Note("one"), new CultRecordHandle<NetworkSchemaNote>(One)).GetAwaiter().GetResult());
Assert.That(store.InAppend.Wait(5000), Is.True);
var clock = Stopwatch.StartNew();
_ = cache.Get<NetworkSchemaNote>(Two);
var waited = clock.Elapsed.TotalMilliseconds;
await writer;
TestContext.Out.WriteLine($"reader of an unrelated key waited {waited:0.0} ms for a 400 ms append");
Assert.That(waited, Is.LessThan(100));
}
// P1: the constructor registers the observer before the journal, so a change stashed by the journal always finds
// its publisher, even while another thread writes the cache during construction.
[Test, Category("SoulPin")]
public void P1_ConstructionUnderConcurrentWritesOrphansNoStash()
{
var orphans = 0;
for (var round = 0; round < 200; round++)
{
var cache = new CultCache();
using var stop = new CancellationTokenSource();
var writer = Task.Run(() =>
{
var i = 0;
while (!stop.IsCancellationRequested)
cache.UpsertAsync(Note("w" + i++), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:race:" + (i % 4)))).GetAwaiter().GetResult();
});
Thread.SpinWait(200 * (round % 7));
var db = Db(cache, true, null);
Thread.SpinWait(2000);
stop.Cancel();
writer.GetAwaiter().GetResult();
orphans += StashCount(db);
db.Dispose();
}
Assert.That(orphans, Is.EqualTo(0), "no stash entry outlives its admission");
}
// P2: every stash entry is removed by its publication.
[Test, Category("SoulPin")]
public async Task P2_TheStashDrainsAfterEveryWrite()
{
var cache = new CultCache();
var db = Db(cache, true, new Store());
await db.PutAsync(One, Note("one"));
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
cache.Commit(batch => batch.Upsert(Note("three"), new CultRecordHandle<NetworkSchemaNote>(Three)));
await db.DeleteAsync<NetworkSchemaNote>(One);
Assert.That(StashCount(db), Is.EqualTo(0));
}
// P3: a replica never applies an entry past a hole. The primary burns 2 with compaction refused, then logs 3; a
// replica at 1 is answered compacted_history and resyncs, never applying 3 on top of 1.
[Test, Category("SoulPin")]
public async Task P3_AReplicaAtTheHoleResyncsRatherThanSkippingIt()
{
var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true };
var cache = new CultCache();
var primary = Db(cache, true, store);
using var peer = new Peer(cache, primary);
var replicaCache = new CultCache();
var replica = Db(replicaCache, false, null);
using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer });
await primary.PutAsync(One, Note("one"));
await replicator.PullOnceAsync(ShardId);
try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { }
await primary.PutAsync(Three, Note("three"));
Assert.That(peer.Log(1).Reason, Is.EqualTo("compacted_history"));
var applied = await replicator.PullOnceAsync(ShardId);
Assert.That(applied, Is.EqualTo(3));
Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get<NetworkSchemaNote>(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" }));
}
[Test, Category("SoulMeasure"), Explicit("Measurement, no assertion; writes a log file.")]
public async Task M1_BareWriteLatencyWithTheFileLogStore()
{
var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N"));
try
{
var cache = new CultCache();
_ = Db(cache, true, new CultNetFileShardMutationLogStore(root));
var times = new List<double>();
for (var i = 0; i < 1000; i++)
{
var clock = Stopwatch.StartNew();
await cache.UpsertAsync(Note("n" + i), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:lat:" + i)));
times.Add(clock.Elapsed.TotalMilliseconds);
}
TestContext.Out.WriteLine($"file store bare write ms: first100 avg {times.Take(100).Average():0.000}, 400-500 avg {times.Skip(400).Take(100).Average():0.000}, last100 avg {times.Skip(900).Average():0.000}, max {times.Max():0.0}");
var bare = new CultCache();
var t2 = new List<double>();
for (var i = 0; i < 1000; i++)
{
var clock = Stopwatch.StartNew();
await bare.UpsertAsync(Note("n" + i), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:lat:" + i)));
t2.Add(clock.Elapsed.TotalMilliseconds);
}
TestContext.Out.WriteLine($"no database bare write ms: last100 avg {t2.Skip(900).Average():0.000}");
}
finally
{
if (Directory.Exists(root)) Directory.Delete(root, true);
}
}
}
}

View file

@ -0,0 +1,142 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
// Soul pass 4c probes. They guard: a committed put whose shard log failed is not answered as a refusal (Q1); reads and watch agree
// on what a database serves (Q2); a replica log apply leaves out a row its shard does not own (Q3); the unowned refusal is not a
// shard-authority exception (Q4); two databases over one cache aggregate their log failures (Q5).
public sealed class SoulP4cProbeTests
{
private const string ShardId = "soulc-shard";
private static readonly CultRecordKey One = new("soulc:one");
private static readonly CultRecordKey Unowned = new("soulc:unowned");
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
private static string SchemaId(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
private sealed class RefusingStore : ICultNetShardMutationLogStore
{
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null) => Array.Empty<CultNetShardLogEntryMessage>();
public void Append(string shardId, CultNetShardLogEntryMessage entry) => throw new IOException("refused " + entry.Sequence);
public long GetCompactedThrough(string shardId) => 0;
public void CompactThrough(string shardId, long sequence) { }
}
private sealed class Logger : GameCult.Logging.ILogger
{
public List<string> Errors { get; } = new();
public void LogInfo(string message) { }
public void LogWarning(string message) { }
public void LogError(string message) => Errors.Add(message);
public void LogDebug(string message) { }
}
private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) =>
new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])],
MutationLogStore = store
});
// Q1: the server's own remote-put path is a writer. A put that committed and could not be logged is answered as a
// failure: the peer is told its write failed, while the cache holds it and it was published.
[Test, Category("SoulFinding")]
public async Task Q1_ServerAnswersACommittedButUnloggedRemotePutAsAFailure()
{
var cache = new CultCache();
var database = Db(cache, primary: true, new RefusingStore());
using var server = new Server(cache, ServerSecurityOptions.Development());
var logger = new Logger();
server.Logger = logger;
using var databaseServer = new CultNetDatabaseServer(server, database);
var handler = (Func<CultNetDocumentPutRawMessage, CultNetServerPeer, Task>)typeof(CultNetDatabaseServer)
.GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
.GetValue(databaseServer)!;
var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer));
var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<NetworkSchemaNote>(One), Note("wire"));
message.ShardId = ShardId;
message.ShardEpoch = 1;
try { await handler(message, peer); } catch (Exception) { }
TestContext.WriteLine("errors: " + string.Join(" | ", logger.Errors));
Assert.That(cache.Get<NetworkSchemaNote>(One)?.Text, Is.EqualTo("wire"), "the put committed");
Assert.That(logger.Errors, Has.None.Contains("raw put failed"), "a committed put must not be reported to the peer as a failed one");
}
// Q2: reads serve a cached row no shard owns, the database watch never publishes its changes: a view built from
// GetAll + Watch (CultMesh Collection, the subscription server's snapshot + WatchAllChanges) goes stale.
[Test, Category("SoulFinding")]
public async Task Q2_ReadsServeAnUnownedRowWhoseChangesTheWatchNeverPublishes()
{
var cache = new CultCache();
var database = Db(cache, primary: true, store: null);
var seen = new List<string>();
database.Watch<MeshQuickstartNote>().Subscribe(change => seen.Add(change.Document?.Body ?? "<removed>"));
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v1" }, new CultRecordHandle<MeshQuickstartNote>(Unowned));
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v2" }, new CultRecordHandle<MeshQuickstartNote>(Unowned));
var read = database.GetAll<MeshQuickstartNote>().Select(note => note.Body).ToArray();
TestContext.WriteLine($"GetAll={string.Join(",", read)} watch={string.Join(",", seen)}");
Assert.That(read.Length == 0 || seen.Count > 0, Is.True, "a row the database reads out must be one whose changes it publishes");
}
// Q3: a replica's log apply door writes a row its shard does not own. The snapshot apply path filters by
// ShardMatchesRawDocument; the log path does not.
[Test, Category("SoulFinding")]
public async Task Q3_AReplicaLogApplyWritesARowItsShardDoesNotOwn()
{
var cache = new CultCache();
var replica = Db(cache, primary: false, store: null);
var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(Unowned), new MeshQuickstartNote { NoteId = "r", Body = "replicated" });
put.ShardId = ShardId;
put.ShardEpoch = 1;
Exception? thrown = null;
try
{
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
MessageId = "log",
ShardId = ShardId,
ShardEpoch = 1,
Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = DateTimeOffset.UtcNow.ToString("O"), ChangeKind = "added", Put = put }]
});
}
catch (Exception exception) { thrown = exception; }
TestContext.WriteLine($"thrown={thrown?.GetType().Name}: {thrown?.Message}; cached={cache.Get<MeshQuickstartNote>(Unowned)?.Body}; log={replica.GetMutationLog(ShardId).Count}");
Assert.That(cache.Get<MeshQuickstartNote>(Unowned), Is.Null, "no shard owns it here, so no path writes it");
}
// Q4 (pin): the unowned refusal is not a shard-authority exception, so the server's routing path does not see it.
[Test, Category("SoulPin")]
public void Q4_TheUnownedRefusalIsNotAShardAuthorityException()
{
var cache = new CultCache();
var database = Db(cache, primary: true, store: null);
var put = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "w" });
var refusal = Assert.ThrowsAsync<CultNetUnownedSchemaException>(async () => await database.ApplyPutAsync(put))!;
Assert.That(refusal, Is.Not.InstanceOf<CultNetShardAuthorityException>());
}
// Q5 (pin): two databases over one cache, both refusing: a writer's catch (CultNetShardLogException) misses.
[Test, Category("SoulPin")]
public void Q5_TwoDatabasesOnOneCacheAggregateTheirLogFailures()
{
var cache = new CultCache();
_ = Db(cache, primary: true, new RefusingStore());
_ = Db(cache, primary: true, new RefusingStore());
var failure = Assert.Throws<AggregateException>(() => cache.Commit(batch =>
batch.Upsert(Note("x"), new CultRecordHandle<NetworkSchemaNote>(One))))!;
Assert.That(failure.InnerExceptions.OfType<CultNetShardLogException>().Count(), Is.EqualTo(2));
}
}
}

View file

@ -0,0 +1,201 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
// Soul pass 4d probes. They guard the replica log relay (a skipped entry is relayed as the primary sent it, in memory and on
// disk), key-prefixed shard reads, and measure two costs; the measurements are explicit.
public sealed class SoulP4dProbeTests
{
private const string ShardId = "soupd-shard";
private static readonly string Now = DateTimeOffset.UtcNow.ToString("O");
private static string NoteSchema(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
private static string MeshSchema(CultCache cache) => cache.Registry.GetRequired<MeshQuickstartNote>().SchemaId;
private static CultNetDatabase Db(CultCache cache, bool primary, string[] schemas, string? keyPrefix = null, ICultNetShardMutationLogStore? store = null) =>
new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: schemas, keyPrefix: keyPrefix)],
MutationLogStore = store
});
private static CultNetShardLogResponseMessage UnownedPutLog(CultNetDatabase anyDb, string key)
{
var put = anyDb.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey(key)), new MeshQuickstartNote { NoteId = "p", Body = "primary-put" });
put.ShardId = ShardId;
put.ShardEpoch = 1;
return new CultNetShardLogResponseMessage
{
MessageId = "log",
ShardId = ShardId,
ShardEpoch = 1,
Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = Now, ChangeKind = "added", Put = put }]
};
}
// Q3a: a replica (no store) that skips an unowned put relays it to a chained replica as a delete.
[Test, Category("SoulFinding")]
public async Task Q3a_MemoryLogRelaysASkippedPutAsARemoval()
{
var cache = new CultCache();
var replica = Db(cache, primary: false, [NoteSchema(cache)]);
await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k"));
var relayed = replica.GetMutationLogMessages(ShardId);
TestContext.Out.WriteLine($"memory relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null} delete={relayed[0].Delete != null}");
Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"), "the relayed entry keeps the primary's kind");
}
// Q3b: the same replica with a durable store relays the primary's put (disk and memory disagree).
[Test, Category("SoulFinding")]
public async Task Q3b_StoreBackedLogRelaysTheSkippedPutAsAPut()
{
var dir = Path.Combine(Path.GetTempPath(), "soulp4d-" + Guid.NewGuid().ToString("N"));
var cache = new CultCache();
var replica = Db(cache, primary: false, [NoteSchema(cache)], store: new CultNetFileShardMutationLogStore(dir));
await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k"));
var relayed = replica.GetMutationLogMessages(ShardId);
var memory = replica.GetMutationLog(ShardId);
TestContext.Out.WriteLine($"store relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null}; memory entry kind={memory[0].Kind} doc={(memory[0].Document == null ? "null" : "set")}");
Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"));
}
// Q3c: a chained replica that does own the row (config drift) and holds it deletes it on the relayed entry.
[Test, Category("SoulFinding")]
public async Task Q3c_AChainedReplicaThatOwnsTheRowDeletesWhatThePrimaryPut()
{
var cacheA = new CultCache();
var a = Db(cacheA, primary: false, [NoteSchema(cacheA)]);
await a.ApplyShardLogResponseAsync(UnownedPutLog(a, "relay:k"));
var cacheC = new CultCache();
var c = Db(cacheC, primary: false, [NoteSchema(cacheC), MeshSchema(cacheC)]);
await cacheC.UpsertAsync(new MeshQuickstartNote { NoteId = "p", Body = "earlier" }, new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey("relay:k")));
await c.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
MessageId = "chain",
ShardId = ShardId,
ShardEpoch = 1,
Entries = a.GetMutationLogMessages(ShardId).ToArray()
});
var row = cacheC.Get<MeshQuickstartNote>(new CultRecordKey("relay:k"));
TestContext.Out.WriteLine($"chained row after relay: {row?.Body ?? "<deleted>"}");
Assert.That(row, Is.Not.Null, "the primary put this row; a relay must not delete it");
}
// A skipped entry is relayed exactly as the primary sent it, whether the replica keeps its log in memory or on disk.
[Test]
public async Task SkippedEntriesAreRelayedIdenticallyFromMemoryAndFromDisk()
{
var dir = Path.Combine(Path.GetTempPath(), "soulp4d-parity-" + Guid.NewGuid().ToString("N"));
try
{
var memoryCache = new CultCache();
var inMemory = Db(memoryCache, primary: false, [NoteSchema(memoryCache)]);
var diskCache = new CultCache();
var onDisk = Db(diskCache, primary: false, [NoteSchema(diskCache)], store: new CultNetFileShardMutationLogStore(dir));
var log = UnownedPutLog(inMemory, "relay:k");
var removal = new CultNetShardLogEntryMessage
{
Sequence = 2,
CommittedAt = Now,
ChangeKind = "removed",
Delete = new CultNetDocumentDeleteMessage
{
MessageId = "d",
SchemaId = memoryCache.Registry.GetRequired<MeshQuickstartNote>().SchemaId,
RecordKey = "relay:k",
ShardId = ShardId,
ShardEpoch = 1
}
};
log.Entries = [log.Entries[0], removal];
await inMemory.ApplyShardLogResponseAsync(log);
await onDisk.ApplyShardLogResponseAsync(log);
var memory = inMemory.GetMutationLogMessages(ShardId);
var disk = onDisk.GetMutationLogMessages(ShardId);
Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(disk.Select(e => e.ChangeKind)));
Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(new[] { true, false }));
Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(disk.Select(e => e.Put != null)));
Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(new[] { false, true }));
Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(disk.Select(e => e.Delete != null)));
Assert.That(memory[0].Put!.Document!.RecordKey, Is.EqualTo(disk[0].Put!.Document!.RecordKey));
Assert.That(memory[0].Put!.Document!.Payload, Is.EqualTo(disk[0].Put!.Document!.Payload), "the document bytes are the primary's");
}
finally
{
if (Directory.Exists(dir)) Directory.Delete(dir, true);
}
}
// Q2: key-prefixed shard: the schema is owned, a key outside the prefix is not.
[Test, Category("SoulPin")]
public async Task Q2_KeyPrefixedReadsServeOnlyTheOwnedKeys()
{
var cache = new CultCache();
var database = Db(cache, primary: true, [NoteSchema(cache)], keyPrefix: "a:");
await cache.UpsertAsync(new NetworkSchemaNote { Text = "a" }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("a:1")));
await cache.UpsertAsync(new NetworkSchemaNote { Text = "b" }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("b:1")));
Assert.That(database.GetAll<NetworkSchemaNote>().Select(n => n.Text), Is.EqualTo(new[] { "a" }));
Assert.That(await database.GetAsync<NetworkSchemaNote>(new CultRecordKey("b:1")), Is.Null);
Assert.That(await database.GetAsync<NetworkSchemaNote>(new CultRecordKey("a:1")), Is.Not.Null);
using var server = new Server(cache, ServerSecurityOptions.Development());
using var databaseServer = new CultNetDatabaseServer(server, database);
var snapshot = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" });
Assert.That(snapshot.Documents.Select(d => d.RecordKey), Is.EqualTo(new[] { "a:1" }));
}
[Test, Category("SoulPin")]
public async Task Q2_PrimaryAllServesEveryCachedRow()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { Shards = [CultNetShardDescriptor.PrimaryAll()] });
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "x", Body = "x" }, new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey("any:1")));
Assert.That(database.GetAll<object>().Count(), Is.EqualTo(1));
Assert.That(database.GetByName<MeshQuickstartNote>("x"), Is.Not.Null);
}
// Q2 cost: GetAll through the database versus the cache, scoped shard (reflection per row).
[Test, Category("SoulMeasure"), Explicit("Measurement, no assertion.")]
public void Q2_GetAllCost()
{
var cache = new CultCache();
var database = Db(cache, primary: false, [NoteSchema(cache)], keyPrefix: "n:");
cache.Commit(batch =>
{
for (var i = 0; i < 20000; i++)
batch.Upsert(new NetworkSchemaNote { Text = "t" + i }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("n:" + i)));
});
_ = cache.GetAll<NetworkSchemaNote>().Count();
_ = database.GetAll<NetworkSchemaNote>().Count();
var sw = Stopwatch.StartNew();
for (var r = 0; r < 5; r++) _ = cache.GetAll<NetworkSchemaNote>().Count();
var cacheMs = sw.Elapsed.TotalMilliseconds / 5;
sw.Restart();
for (var r = 0; r < 5; r++) _ = database.GetAll<NetworkSchemaNote>().Count();
var dbMs = sw.Elapsed.TotalMilliseconds / 5;
TestContext.Out.WriteLine($"GetAll 20000 rows: cache={cacheMs:F2}ms database={dbMs:F2}ms ratio={dbMs / Math.Max(cacheMs, 0.001):F1}x");
}
// Wire: the bytes a peer receives for the unowned refusal.
[Test, Category("SoulMeasure"), Explicit("Prints the wire bytes the Rust captured-bytes case pins.")]
public void Wire_UnownedSchemaBytes()
{
var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")));
var bytes = MessagePack.MessagePackSerializer.Serialize(message, CultNetSchemaMessageSerialization.Options);
TestContext.Out.WriteLine("HEX=" + Convert.ToHexString(bytes));
TestContext.Out.WriteLine("JSON=" + MessagePack.MessagePackSerializer.ConvertToJson(bytes));
}
}
}

View file

@ -0,0 +1,276 @@
#nullable enable
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using GameCult.Caching;
using NUnit.Framework;
using R3;
using static GameCult.Networking.Tests.NetworkingTests;
namespace GameCult.Networking.Tests
{
[CultDocument("tests.soul_throwing_name", "tests.soul_throwing_name.v1")]
[MessagePack.MessagePackObject]
public sealed class SoulThrowingName
{
[MessagePack.Key(0)]
public string Text { get; set; } = string.Empty;
[MessagePack.Key(1)]
[CultName]
public string Name
{
get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text;
set { }
}
}
[NonParallelizable]
public sealed class SoulPub3Tests
{
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
private static string SchemaId => CultDocumentRegistry.Shared.GetRequired<NetworkSchemaNote>().SchemaId;
// Another thread becomes the drainer and parks inside a subscriber on the change "slow".
private static (Task Writer, ManualResetEventSlim Release) BlockDrainer(CultNetDatabase database, CultCache cache, out IDisposable subscription)
{
var entered = new ManualResetEventSlim();
var release = new ManualResetEventSlim();
subscription = database.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "slow")
{
entered.Set();
release.Wait(Wait);
}
});
var writer = Task.Run(() => cache.UpsertAsync(Note("slow"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul3:slow"))).GetAwaiter().GetResult());
Assert.That(entered.Wait(Wait), Is.True, "drainer parked");
return (writer, release);
}
// P1: a write that returns has been logged and published.
[Test]
public async Task P1_AWriteThatReturnedIsLoggedAndPublished()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var seen = new ConcurrentQueue<(string? Text, int Thread)>();
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Document?.Text, Environment.CurrentManagedThreadId)));
var (writer, release) = BlockDrainer(database, cache, out var block);
var fast = new CultRecordKey("soul3:fast");
var writerThread = 0;
await Task.Run(async () => { writerThread = Environment.CurrentManagedThreadId; await database.PutAsync(fast, Note("fast")); });
var logged = database.LastWriteSequence(SchemaId, fast);
var published = seen.Any(entry => entry.Text == "fast");
release.Set();
await writer;
block.Dispose();
var deliveredOn = seen.First(entry => entry.Text == "fast").Thread;
TestContext.Progress.WriteLine($"SOUL P1 logged-at-return={logged} published-at-return={published} writer={writerThread} deliveredOn={deliveredOn}");
Assert.Multiple(() =>
{
Assert.That(logged, Is.Not.Null, "PutAsync returned before its change was logged");
Assert.That(published, Is.True, "PutAsync returned before its change was published");
});
}
// P2: a subscriber that waits (bounded) for another thread's write to be observed.
[Test]
public async Task P2_ASubscriberWaitingForAnotherThreadsWriteIsNotStarved()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var other = new ManualResetEventSlim();
var observedInTime = false;
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "other")
other.Set();
if (change.Document?.Text == "trigger")
{
Task.Run(() => database.PutAsync(new CultRecordKey("soul3:other"), Note("other")));
observedInTime = other.Wait(TimeSpan.FromSeconds(2));
}
});
await database.PutAsync(new CultRecordKey("soul3:trigger"), Note("trigger"));
Assert.That(observedInTime, Is.True, "another thread's committed write never reached subscribers while this handler ran");
}
// P3: one observer's escaped exception (fail-fast R3 handler) loses one change; it must not stall all later ones.
[Test]
public async Task P3_AnEscapedObserverExceptionDoesNotStallLaterChanges()
{
var previous = ObservableSystem.GetUnhandledExceptionHandler();
ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception);
try
{
var cache = new CultCache();
using var thrower = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom");
});
var database = new CultNetDatabase(cache);
var seen = new ConcurrentQueue<string?>();
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue(change.Document?.Text));
try { await cache.UpsertAsync(Note("boom"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul3:boom"))); }
catch (InvalidOperationException) { }
ObservableSystem.RegisterUnhandledExceptionHandler(previous);
var after = new CultRecordKey("soul3:after");
await database.PutAsync(after, Note("after"));
Assert.Multiple(() =>
{
Assert.That(seen, Does.Contain("after"), "a later commit was never published");
Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged");
});
}
finally
{
ObservableSystem.RegisterUnhandledExceptionHandler(previous);
}
}
// P4: an admission that throws inside Apply after minting a Sequence (a [CultName] getter) must not stall later changes.
[Test]
public async Task P4_AThrowMidApplyDoesNotStallLaterChanges()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var seen = new ConcurrentQueue<string?>();
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue(change.Document?.Text));
try
{
cache.Commit(batch =>
{
batch.Upsert(new SoulThrowingName { Text = "ok" }, new CultRecordHandle<SoulThrowingName>(new CultRecordKey("soul3:ok")));
batch.Upsert(new SoulThrowingName { Text = "boom" }, new CultRecordHandle<SoulThrowingName>(new CultRecordKey("soul3:boomname")));
});
}
catch (InvalidOperationException) { }
var after = new CultRecordKey("soul3:after-apply");
await database.PutAsync(after, Note("after"));
TestContext.Progress.WriteLine($"SOUL P4 ok-in-cache={cache.Get(new CultRecordKey("soul3:ok")) != null}");
Assert.Multiple(() =>
{
Assert.That(seen, Does.Contain("after"), "a later commit was never published");
Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged");
});
}
// P5: writers no longer carry their own delivery; the backlog behind one drainer grows without bound.
[Test]
public async Task P5_OtherWritersDoNotPileAnUnboundedBacklogOnOneDrainer()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache);
var delivered = 0;
var backlogAtBDone = -1;
const int bWrites = 2000;
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change =>
{
if (change.Document?.Text?.StartsWith("b") == true) Interlocked.Increment(ref delivered);
if (change.Document?.Text == "a")
{
Task.Run(async () =>
{
for (var i = 0; i < bWrites; i++)
await database.PutAsync(new CultRecordKey($"soul3:b:{i}"), Note($"b{i}"));
}).Wait(Wait);
backlogAtBDone = bWrites - Volatile.Read(ref delivered);
}
});
await database.PutAsync(new CultRecordKey("soul3:a"), Note("a"));
TestContext.Progress.WriteLine($"SOUL P5 undelivered after B's {bWrites} writes returned: {backlogAtBDone}");
Assert.That(backlogAtBDone, Is.EqualTo(0));
}
// P6: a prediction still queued when the authoritative put arrives.
[Test]
public async Task P6_AnAuthoritativePutArrivingBeforeItsPredictionIsDeliveredStillReconciles()
{
var cache = new CultCache();
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
{
RuntimeId = "local",
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
});
var key = new CultRecordKey("soul3:predict");
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
using var watch = database.WatchRecord<NetworkSchemaNote>(key).Subscribe(change => kinds.Enqueue(change.Kind));
CultNetDocumentPutRawMessage Put(string text)
{
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
message.ShardId = database.Shards[0].ShardId;
message.ShardEpoch = database.Shards[0].Epoch;
return message;
}
var (writer, release) = BlockDrainer(database, cache, out var block);
await database.PutPredictedAsync(key, Note("predicted"));
await database.ApplyPutAsync(Put("auth1"));
release.Set();
await writer;
block.Dispose();
await database.ApplyPutAsync(Put("auth2"));
Assert.That(kinds, Is.EqualTo(new[]
{
CultNetDatabaseChangeKind.Predicted,
CultNetDatabaseChangeKind.Reconciled,
CultNetDatabaseChangeKind.Updated
}));
}
// P7: a replica's absent-key delete is logged ahead of an earlier entry still queued; a bounded pull then sees a gap.
[Test]
public async Task P7_AReplicaLogStaysInSequenceOrderWhenAnAbsentDeleteOvertakesAQueuedPut()
{
const string shardId = "chain3";
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
{
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [SchemaId])]
});
var p = Make(new CultCache(), true);
var rCache = new CultCache();
var r = Make(rCache, false);
var c = Make(new CultCache(), false);
async Task Pull(CultNetDatabase from, CultNetDatabase to, int? limit = null)
{
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
{
ShardId = shardId,
ShardEpoch = 2,
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId), limit).ToArray()
});
}
var z = new CultRecordKey("soul3:z");
await p.PutAsync(z, Note("z"));
await Pull(p, r);
await Pull(r, c);
rCache.Remove(z);
await p.PutAsync(new CultRecordKey("soul3:a"), Note("a"));
await p.DeleteAsync<NetworkSchemaNote>(z);
var (writer, release) = BlockDrainer(r, rCache, out var block);
await Pull(p, r);
release.Set();
await writer;
block.Dispose();
TestContext.Progress.WriteLine($"SOUL P7 r log order: {string.Join(",", r.GetMutationLog(shardId).Select(e => e.Sequence))}");
Assert.Multiple(() =>
{
Assert.That(r.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }));
Assert.DoesNotThrowAsync(() => Pull(r, c, limit: 1), "a bounded pull from R");
});
}
}
}