Merge publication Cut 4: CultNetDatabase rebuilt on the cache journal
Kind, prediction and log are decided in the journal under the cache gate; the reorder buffer, drainer and prediction gate are gone. Q-P5 A: one typed CultNetShardLogException per journal call, after publication. The four old log bugs are fixed. No shard owns a schema means no write on any path (typed refusal, wire code unowned_schema) and no row served; replicas relay skipped entries unchanged. Four Soul passes; follow-ups recorded in docs/cultcache-publication-cut.md (crash-safe log store is a hard prerequisite before any live daemon enables durable shard logs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
5b42eb0250
24 changed files with 3732 additions and 349 deletions
|
|
@ -104,6 +104,16 @@ Not implemented yet:
|
|||
- cross-runtime rollback/resimulation helpers for simulation frames
|
||||
- declared CRDT merge policies
|
||||
|
||||
## Error Codes
|
||||
|
||||
`cultnet.error.v0` carries an optional machine-readable `code`. The set of codes is a closed enum
|
||||
(`cultnet.error.schema.json`): Rust and TypeScript refuse a message whose code they do not know, they do
|
||||
not guess. Adding a code is therefore a coordinated change across every runtime, made together: the schema (both
|
||||
copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in
|
||||
`packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are
|
||||
`selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported` and
|
||||
`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns).
|
||||
|
||||
## Live Invariants
|
||||
|
||||
- CultCache owns document schema, identity, local indexes, and local diffing.
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@
|
|||
}
|
||||
},
|
||||
"code": {
|
||||
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.",
|
||||
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).",
|
||||
"type": ["string", "null"],
|
||||
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"]
|
||||
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"]
|
||||
},
|
||||
"details": {
|
||||
"description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.",
|
||||
|
|
@ -36,8 +36,8 @@
|
|||
"additionalProperties": false,
|
||||
"required": ["field", "value", "asOf", "current"],
|
||||
"properties": {
|
||||
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] },
|
||||
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] },
|
||||
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] },
|
||||
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] },
|
||||
"asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 },
|
||||
"current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@
|
|||
}
|
||||
},
|
||||
"code": {
|
||||
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.",
|
||||
"description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).",
|
||||
"type": ["string", "null"],
|
||||
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"]
|
||||
"enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"]
|
||||
},
|
||||
"details": {
|
||||
"description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.",
|
||||
|
|
@ -36,8 +36,8 @@
|
|||
"additionalProperties": false,
|
||||
"required": ["field", "value", "asOf", "current"],
|
||||
"properties": {
|
||||
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] },
|
||||
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] },
|
||||
"field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] },
|
||||
"value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] },
|
||||
"asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 },
|
||||
"current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -250,8 +250,8 @@ impl CultNetShardDescriptor {
|
|||
}
|
||||
}
|
||||
|
||||
/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings
|
||||
/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth
|
||||
/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the strings
|
||||
/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a
|
||||
/// string it does not know refuses to decode rather than guess a meaning for it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
|
|
@ -261,6 +261,7 @@ pub enum CultNetErrorCode {
|
|||
CursorInvalid,
|
||||
ReferenceOutsideTarget,
|
||||
VariantUnsupported,
|
||||
UnownedSchema,
|
||||
}
|
||||
|
||||
impl CultNetErrorCode {
|
||||
|
|
@ -271,6 +272,7 @@ impl CultNetErrorCode {
|
|||
Self::CursorInvalid => "cursor_invalid",
|
||||
Self::ReferenceOutsideTarget => "reference_outside_target",
|
||||
Self::VariantUnsupported => "variant_unsupported",
|
||||
Self::UnownedSchema => "unowned_schema",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -281,6 +283,7 @@ impl CultNetErrorCode {
|
|||
"cursor_invalid" => Ok(Self::CursorInvalid),
|
||||
"reference_outside_target" => Ok(Self::ReferenceOutsideTarget),
|
||||
"variant_unsupported" => Ok(Self::VariantUnsupported),
|
||||
"unowned_schema" => Ok(Self::UnownedSchema),
|
||||
other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -118,6 +118,46 @@ fn cases() -> Vec<Case> {
|
|||
details: None,
|
||||
},
|
||||
},
|
||||
Case {
|
||||
name: "variant_unsupported_subscription_ended",
|
||||
// C#: ForVariantUnsupported("variant unsupported here", "sub-1")
|
||||
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0",
|
||||
message: || CultNetMessage::Error {
|
||||
error: "variant unsupported here".to_string(),
|
||||
code: Some(CultNetErrorCode::VariantUnsupported),
|
||||
details: Some(CultNetErrorDetails {
|
||||
field: Some("subscriptionId".to_string()),
|
||||
value: Some("sub-1".to_string()),
|
||||
as_of: None,
|
||||
current: None,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Case {
|
||||
name: "variant_unsupported_no_subscription",
|
||||
// C#: ForVariantUnsupported("variant unsupported here") - no subscription ended, so Details stays null.
|
||||
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0",
|
||||
message: || CultNetMessage::Error {
|
||||
error: "variant unsupported here".to_string(),
|
||||
code: Some(CultNetErrorCode::VariantUnsupported),
|
||||
details: None,
|
||||
},
|
||||
},
|
||||
Case {
|
||||
name: "unowned_schema",
|
||||
// C#: ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")))
|
||||
csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0",
|
||||
message: || CultNetMessage::Error {
|
||||
error: "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.".to_string(),
|
||||
code: Some(CultNetErrorCode::UnownedSchema),
|
||||
details: Some(CultNetErrorDetails {
|
||||
field: Some("schemaId".to_string()),
|
||||
value: Some("some.schema.v1".to_string()),
|
||||
as_of: None,
|
||||
current: None,
|
||||
}),
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,10 +47,20 @@ which route wrote it.
|
|||
- Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a
|
||||
per-cache, in-memory number assigned when the cache admits the change, under
|
||||
its gate. It increases in admission order and is not persisted.
|
||||
`OnUpdate(previous, current)` carries none. Streams CultNet derives from
|
||||
`OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server,
|
||||
database-backed Mesh handles) have no ordering guarantee and no stale
|
||||
protection.
|
||||
`OnUpdate(previous, current)` carries none. CultNet derives its streams
|
||||
(`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed
|
||||
Mesh handles) from `Watch` plus one journal (`AddJournal`), so a load, a
|
||||
commit and a variant dependent's re-resolution all reach subscribers and the
|
||||
mutation log. The journal runs under the gate in `Sequence` order, so the
|
||||
shard log is written in cache order; the `Watch` observer publishes what the
|
||||
journal decided. A database write door only hands the journal the wire message
|
||||
or replicated entry for the change it admitted. A replica publishes but never
|
||||
mints log sequences. A primary logs every change it admits, loads and pulls
|
||||
included; a store attached after the database exists has its hydration logged
|
||||
too. A primary that cannot log a change compacts through that change's
|
||||
sequence and throws `CultNetShardLogException` to the writer after publication:
|
||||
the commit stands, and a replica behind it resynchronizes from a snapshot. The
|
||||
streams carry no `Sequence` and have no stale protection.
|
||||
- `GetWithSequence(key)` returns the document and the cache's current
|
||||
`Sequence`, read together under the gate: every change with a `Sequence` at
|
||||
or below it is reflected in the document.
|
||||
|
|
@ -126,6 +136,10 @@ which route wrote it.
|
|||
needs the cache or its gate: another thread's write, a lock a writer holds.
|
||||
Its own leaf locks, taken only inside the journal, are the only locks that may
|
||||
nest under the gate.
|
||||
- A shard mutation log store (`ICultNetShardMutationLogStore`) runs inside the
|
||||
CultNet journal, under the cache's gate: it must not call the cache or wait
|
||||
for a thread that does. A store that writes its own cache throws through the
|
||||
journal's reentry guard, and the admission stands.
|
||||
- A thread holds one cache's gate at a time: entering a cache's hold inside
|
||||
another cache's hold throws.
|
||||
- A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate,
|
||||
|
|
|
|||
|
|
@ -2124,7 +2124,15 @@ namespace GameCult.Caching
|
|||
Remove(handle.Key);
|
||||
}
|
||||
|
||||
// false: a condition failed; nothing was written, changed in memory, or published.
|
||||
/// <summary>
|
||||
/// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer
|
||||
/// receives the failure. One failure arrives as itself, several as an <see cref="AggregateException"/>. A writer that
|
||||
/// wants a particular failure (a CultNetShardLogException, when a database's log refused the commit) catches that type,
|
||||
/// or unwraps <see cref="AggregateException.InnerExceptions"/> and looks for it.
|
||||
/// </remarks>
|
||||
public bool Commit(Action<CultCacheBatch> stage)
|
||||
{
|
||||
return Land(stage, wait: true) == CultCommitOutcome.Committed;
|
||||
|
|
@ -2427,7 +2435,8 @@ namespace GameCult.Caching
|
|||
// run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not
|
||||
// guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate,
|
||||
// in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even
|
||||
// if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown.
|
||||
// if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their
|
||||
// origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions.
|
||||
// If the body threw, its exception wins.
|
||||
internal T Held<T>(Func<T> body)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3379,7 +3379,8 @@ namespace GameCult.Mesh
|
|||
if (document != null)
|
||||
return document;
|
||||
|
||||
var untyped = database.Cache.Get(key);
|
||||
// Through the database, which serves only what its shards own; the cache behind it may hold more.
|
||||
var untyped = await database.GetAsync<object>(key).ConfigureAwait(false);
|
||||
if (untyped != null && IsSameCultDocumentSchema<TDocument>(untyped.GetType()))
|
||||
return ConvertUntypedDocument<TDocument>(untyped);
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -112,10 +112,13 @@ namespace GameCult.Networking
|
|||
request);
|
||||
}
|
||||
|
||||
// A database serves only what its shards own.
|
||||
return _database.Documents.CreateRawSnapshotResponse(
|
||||
_database.Cache,
|
||||
string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId,
|
||||
request);
|
||||
request,
|
||||
options: null,
|
||||
rowFilter: _database.Owns);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -135,6 +138,9 @@ namespace GameCult.Networking
|
|||
// R-Q: asOf is one shard-log watermark, so a selection whose matched rows span more than
|
||||
// one shard's log is refused rather than answered against a watermark that is not
|
||||
// exact for all of them.
|
||||
// A row whose schema no shard owns is not part of what this database serves: it is left out, not an error.
|
||||
options: null,
|
||||
rowFilter: _database.Owns,
|
||||
shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId,
|
||||
cursorKey: _database.CursorKey,
|
||||
// S-9: once the matched rows resolve to one shard, asOf is that shard's own watermark,
|
||||
|
|
@ -312,6 +318,16 @@ namespace GameCult.Networking
|
|||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
}
|
||||
}
|
||||
catch (CultNetUnownedSchemaException ex)
|
||||
{
|
||||
_server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}");
|
||||
peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex));
|
||||
}
|
||||
catch (Exception ex) when (CommittedWithLogFailure(ex))
|
||||
{
|
||||
// The write committed and was published; only its shard log failed, so the peer is not told it was refused.
|
||||
_server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_server.Logger.LogError($"CultNet raw put failed: {ex.Message}");
|
||||
|
|
@ -332,6 +348,15 @@ namespace GameCult.Networking
|
|||
peer.SendCultNet(CreateRoutingError(ex));
|
||||
}
|
||||
}
|
||||
catch (CultNetUnownedSchemaException ex)
|
||||
{
|
||||
_server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}");
|
||||
peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex));
|
||||
}
|
||||
catch (Exception ex) when (CommittedWithLogFailure(ex))
|
||||
{
|
||||
_server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
|
||||
|
|
@ -604,6 +629,12 @@ namespace GameCult.Networking
|
|||
: messageId;
|
||||
}
|
||||
|
||||
// A CultNetShardLogException means the write committed and was published (the writer hears of a failed log afterwards);
|
||||
// the cache wraps it in an AggregateException when something else failed in the same hold.
|
||||
private static bool CommittedWithLogFailure(Exception exception) =>
|
||||
exception is CultNetShardLogException ||
|
||||
exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException);
|
||||
|
||||
private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception)
|
||||
{
|
||||
return new CultNetErrorMessage
|
||||
|
|
|
|||
|
|
@ -421,7 +421,8 @@ namespace GameCult.Networking
|
|||
documentSelection,
|
||||
ordinalOf: (schemaId, key) => _database.LastWriteSequence(schemaId, key) ?? 0,
|
||||
asOf: _database.CurrentAsOf(),
|
||||
options: null);
|
||||
options: null,
|
||||
rowFilter: _database.Owns);
|
||||
// The snapshot is one row per record key by construction, so the duplicate-key throw this
|
||||
// used to guard can no longer fire (docs/cultnet-selection-cut.md, S2-2); deleted rather
|
||||
// than tested around.
|
||||
|
|
|
|||
|
|
@ -280,7 +280,16 @@ namespace GameCult.Networking
|
|||
CultCache cache,
|
||||
string messageId,
|
||||
CultNetSnapshotRequestMessage? filter = null,
|
||||
CultNetDocumentMessageOptions? options = null)
|
||||
CultNetDocumentMessageOptions? options = null) =>
|
||||
CreateRawSnapshotResponse(cache, messageId, filter, options, rowFilter: null);
|
||||
|
||||
// rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own).
|
||||
internal CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse(
|
||||
CultCache cache,
|
||||
string messageId,
|
||||
CultNetSnapshotRequestMessage? filter,
|
||||
CultNetDocumentMessageOptions? options,
|
||||
Func<CultDocumentDescriptor, CultRecordKey, bool>? rowFilter)
|
||||
{
|
||||
if (cache == null) throw new ArgumentNullException(nameof(cache));
|
||||
var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds);
|
||||
|
|
@ -323,7 +332,7 @@ namespace GameCult.Networking
|
|||
ordinalOf = static (_, _) => 0;
|
||||
}
|
||||
|
||||
var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options);
|
||||
var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options, rowFilter);
|
||||
|
||||
return new CultNetSnapshotResponseRawMessage
|
||||
{
|
||||
|
|
@ -348,13 +357,27 @@ namespace GameCult.Networking
|
|||
CultNetDocumentMessageOptions? options = null,
|
||||
Func<string, CultRecordKey, string>? shardIdOf = null,
|
||||
CultNetSelectionCursorKey? cursorKey = null,
|
||||
Func<string, ulong>? asOfForShard = null)
|
||||
Func<string, ulong>? asOfForShard = null) =>
|
||||
CreateSelectionResponse(cache, messageId, selection, ordinalOf, asOf, options, shardIdOf, cursorKey, asOfForShard, rowFilter: null);
|
||||
|
||||
// rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own).
|
||||
internal CultNetSnapshotResponseRawV1Message CreateSelectionResponse(
|
||||
CultCache cache,
|
||||
string messageId,
|
||||
CultNetSelection selection,
|
||||
Func<string, CultRecordKey, long> ordinalOf,
|
||||
ulong asOf,
|
||||
CultNetDocumentMessageOptions? options,
|
||||
Func<string, CultRecordKey, string>? shardIdOf,
|
||||
CultNetSelectionCursorKey? cursorKey,
|
||||
Func<string, ulong>? asOfForShard,
|
||||
Func<CultDocumentDescriptor, CultRecordKey, bool>? rowFilter)
|
||||
{
|
||||
if (cache == null) throw new ArgumentNullException(nameof(cache));
|
||||
if (selection == null) throw new ArgumentNullException(nameof(selection));
|
||||
if (ordinalOf == null) throw new ArgumentNullException(nameof(ordinalOf));
|
||||
|
||||
var page = SelectPage(cache, selection, ordinalOf, asOf, options, shardIdOf: shardIdOf, cursorKey: cursorKey, asOfForShard: asOfForShard);
|
||||
var page = SelectPage(cache, selection, ordinalOf, asOf, options, rowFilter, shardIdOf, cursorKey, asOfForShard);
|
||||
return new CultNetSnapshotResponseRawV1Message
|
||||
{
|
||||
MessageId = RequireNonEmpty(messageId, nameof(messageId)),
|
||||
|
|
|
|||
|
|
@ -511,7 +511,7 @@ namespace GameCult.Networking
|
|||
[Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the machine-readable refusal code (R-N), e.g. <c>selection_invalid</c>,
|
||||
/// <c>cursor_stale</c>, <c>cursor_invalid</c>, <c>reference_outside_target</c>. Additive to
|
||||
/// <c>cursor_stale</c>, <c>cursor_invalid</c>, <c>reference_outside_target</c>, <c>unowned_schema</c>. Additive to
|
||||
/// <c>cultnet.error.v0</c>; a v0 peer that does not know this field ignores it.
|
||||
/// </summary>
|
||||
[Key("code")] public string? Code { get; set; }
|
||||
|
|
@ -532,6 +532,17 @@ namespace GameCult.Networking
|
|||
Details = new CultNetErrorDetails { Field = ex.Field, Value = ex.Value }
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Builds the wire error for a write no shard owns: code <c>unowned_schema</c>, details <c>{ field: "schemaId", value }</c>.
|
||||
/// </summary>
|
||||
public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) =>
|
||||
new CultNetErrorMessage
|
||||
{
|
||||
Error = $"unowned_schema: {ex.Message}",
|
||||
Code = "unowned_schema",
|
||||
Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId }
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Builds the wire error for a cursor refusal (R-N): code <c>cursor_stale</c> with details
|
||||
/// <c>{ asOf, current }</c>, or code <c>cursor_invalid</c> with no details.
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ namespace GameCult.Networking
|
|||
/// <summary>
|
||||
/// Stores accepted shard-log entries in their replica catch-up wire shape.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of
|
||||
/// the cache and blocks its readers too, because reads take the same gate. A store must not call the cache, or wait for a thread that does. Appends must throw rather than
|
||||
/// swallow a failure: the database burns the sequence and compacts past it, and tells the writer.
|
||||
/// </remarks>
|
||||
public interface ICultNetShardMutationLogStore
|
||||
{
|
||||
/// <summary>
|
||||
|
|
|
|||
|
|
@ -938,6 +938,32 @@ public sealed class CultMeshStreamingTests
|
|||
await act.Should().ThrowAsync<NotSupportedException>();
|
||||
}
|
||||
|
||||
// A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type
|
||||
// out of the cache behind it.
|
||||
[TestCase("tests.mesh_note.v1", true)]
|
||||
[TestCase("tests.unrelated.v1", false)]
|
||||
public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served)
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var key = new CultRecordKey("note:one");
|
||||
await cache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cached" }, new CultRecordHandle<MeshNoteDocument>(key));
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "reader",
|
||||
Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) }
|
||||
});
|
||||
var handle = CultMesh.Document<MeshNoteAliasDocument>(database, key, CultMesh.Verse("starbridge", "reader"));
|
||||
|
||||
if (served)
|
||||
{
|
||||
(await handle.LatestAsync()).Text.Should().Be("cached");
|
||||
}
|
||||
else
|
||||
{
|
||||
await FluentActions.Awaiting(() => handle.LatestAsync()).Should().ThrowAsync<KeyNotFoundException>();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DocumentHandle_SubmitsPredictionsThroughCultNetDatabase()
|
||||
{
|
||||
|
|
|
|||
1199
tests/GameCult.Networking.Tests/CultNetJournalTests.cs
Normal file
1199
tests/GameCult.Networking.Tests/CultNetJournalTests.cs
Normal file
File diff suppressed because it is too large
Load diff
351
tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs
Normal file
351
tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
public sealed class CultNetPublicationOrderTests
|
||||
{
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
|
||||
private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text;
|
||||
|
||||
// A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed
|
||||
// (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile.
|
||||
[Test]
|
||||
public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var entered = new ManualResetEventSlim();
|
||||
var release = new ManualResetEventSlim();
|
||||
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "A")
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(Wait);
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache);
|
||||
var seen = new List<string?>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); });
|
||||
var key = new CultRecordKey("soul:order");
|
||||
|
||||
var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult());
|
||||
Assert.That(entered.Wait(Wait), Is.True);
|
||||
await database.PutAsync(key, Note("B"));
|
||||
release.Set();
|
||||
await a;
|
||||
|
||||
var shard = database.Shards[0].ShardId;
|
||||
var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last();
|
||||
|
||||
// A replica built from the log.
|
||||
var replicaCache = new CultCache();
|
||||
var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)]
|
||||
});
|
||||
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = shard,
|
||||
ShardEpoch = database.Shards[0].Epoch,
|
||||
Entries = database.GetMutationLogMessages(shard).ToArray()
|
||||
});
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B");
|
||||
Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write");
|
||||
Assert.That(replicaCache.Get<NetworkSchemaNote>(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary");
|
||||
// Delivery is not ordered across threads (order is data: a wire peer drops the stale change by Sequence, Cut 6),
|
||||
// so a subscriber sees both changes, once each, in either order.
|
||||
Assert.That(seen, Is.EquivalentTo(new[] { "A", "B" }), "each change is published exactly once");
|
||||
});
|
||||
}
|
||||
|
||||
// Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache.
|
||||
[Test]
|
||||
public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing()
|
||||
{
|
||||
var divergent = 0;
|
||||
const int rounds = 300;
|
||||
for (var round = 0; round < rounds; round++)
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("soul:hammer");
|
||||
using var start = new Barrier(2);
|
||||
Parallel.For(0, 2, writer =>
|
||||
{
|
||||
start.SignalAndWait();
|
||||
for (var i = 0; i < 40; i++)
|
||||
cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle<NetworkSchemaNote>(key)).GetAwaiter().GetResult();
|
||||
});
|
||||
var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last();
|
||||
if (!ReferenceEquals(last.Document, cache.Get<NetworkSchemaNote>(key)))
|
||||
divergent++;
|
||||
}
|
||||
|
||||
TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}");
|
||||
Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache");
|
||||
}
|
||||
|
||||
// A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in
|
||||
// the window before the door's own change is handled, consumes the door's context.
|
||||
[Test]
|
||||
public async Task S2_ARemovalNeverWearsAReconcilingPutsContext()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var entered = new ManualResetEventSlim();
|
||||
var release = new ManualResetEventSlim();
|
||||
using var widen = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "authoritative")
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(Wait);
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:reconcile");
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text)));
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
var a = Task.Run(() => database.ApplyPutAsync(message));
|
||||
Assert.That(entered.Wait(Wait), Is.True);
|
||||
cache.Remove(key);
|
||||
release.Set();
|
||||
await a;
|
||||
|
||||
var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1),
|
||||
"the removal is published as a removal: " + string.Join(", ", published));
|
||||
Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text),
|
||||
Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document");
|
||||
Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True,
|
||||
"no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document))));
|
||||
});
|
||||
}
|
||||
|
||||
// A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not
|
||||
// hold (B removed it locally, a supported write) must still keep B's log contiguous for C.
|
||||
[Test]
|
||||
public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey()
|
||||
{
|
||||
const string shardId = "chain";
|
||||
var schemaId = new CultCache().Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])]
|
||||
});
|
||||
var p = Make(new CultCache(), true);
|
||||
var bCache = new CultCache();
|
||||
var b = Make(bCache, false);
|
||||
var c = Make(new CultCache(), false);
|
||||
async Task Pull(CultNetDatabase from, CultNetDatabase to)
|
||||
{
|
||||
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = shardId,
|
||||
ShardEpoch = 2,
|
||||
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray()
|
||||
});
|
||||
}
|
||||
|
||||
var k = new CultRecordKey("soul:chain:k");
|
||||
await p.PutAsync(k, Note("x"));
|
||||
await Pull(p, b);
|
||||
await Pull(b, c);
|
||||
bCache.Remove(k);
|
||||
await p.DeleteAsync<NetworkSchemaNote>(k);
|
||||
await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y"));
|
||||
await Pull(p, b);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }),
|
||||
"B's log holds every primary entry it applied");
|
||||
Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap");
|
||||
});
|
||||
}
|
||||
|
||||
// A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction.
|
||||
[Test]
|
||||
public async Task S4_APredictionMadeInReactionToAReconcileSurvives()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:repredict");
|
||||
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
|
||||
var repredicted = false;
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
kinds.Enqueue(change.Kind);
|
||||
if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted)
|
||||
{
|
||||
repredicted = true;
|
||||
database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult();
|
||||
}
|
||||
});
|
||||
CultNetDocumentPutRawMessage Put(string text)
|
||||
{
|
||||
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
return message;
|
||||
}
|
||||
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
await database.ApplyPutAsync(Put("auth1"));
|
||||
await database.ApplyPutAsync(Put("auth2"));
|
||||
|
||||
Assert.That(kinds, Is.EqualTo(new[]
|
||||
{
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
CultNetDatabaseChangeKind.Reconciled,
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
CultNetDatabaseChangeKind.Reconciled
|
||||
}));
|
||||
}
|
||||
// Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A
|
||||
// refused prediction leaves nothing behind that a later write of the same instance would wear.
|
||||
[Test]
|
||||
public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<CultNetDocumentRegistryVariantTests.VariantWireFixture>(registry));
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = documents,
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var baseKey = new CultRecordKey("soul-refuse-base");
|
||||
var variantKey = new CultRecordKey("soul-refuse-variant");
|
||||
await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 },
|
||||
new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<CultNetDocumentRegistryVariantTests.VariantWireFixture>(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v")
|
||||
});
|
||||
var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 };
|
||||
Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException);
|
||||
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
|
||||
using var subscriber = database.Watch<CultNetDocumentRegistryVariantTests.VariantWireFixture>().Subscribe(change => kinds.Enqueue(change.Kind));
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
|
||||
await cache.UpsertAsync(instance, new CultRecordHandle<CultNetDocumentRegistryVariantTests.VariantWireFixture>(new CultRecordKey("soul-refuse-other")));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1));
|
||||
});
|
||||
}
|
||||
|
||||
// Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a
|
||||
// durable log store can observe even though the disposed Subject hides publication.
|
||||
[Test]
|
||||
public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog()
|
||||
{
|
||||
var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N"));
|
||||
var store = new CultNetFileShardMutationLogStore(root);
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store });
|
||||
var shard = database.Shards[0].ShardId;
|
||||
await cache.UpsertAsync(Note("before"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:before")));
|
||||
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
|
||||
database.Dispose();
|
||||
|
||||
try { await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:dispose:after"))); }
|
||||
catch { /* a throwing handler would also be a failure below */ }
|
||||
|
||||
Assert.That(store.Read(shard), Has.Count.EqualTo(1));
|
||||
System.IO.Directory.Delete(root, true);
|
||||
}
|
||||
|
||||
// The log entry exists before its change is published: a subscriber that reads the log from its handler finds it.
|
||||
[Test]
|
||||
public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("soul:log-first");
|
||||
var loggedWhenPublished = new ConcurrentQueue<bool>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value)));
|
||||
|
||||
await database.PutAsync(key, Note("one"));
|
||||
cache.Remove(key);
|
||||
|
||||
Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true }));
|
||||
}
|
||||
|
||||
// An observer that mutates the admitted instance in place and upserts it again writes a second change of the same
|
||||
// instance. It is published as its own update and never wears the context of the door's change, even when it
|
||||
// reaches the database first.
|
||||
[Test]
|
||||
public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "authoritative")
|
||||
{
|
||||
change.Document.Text = "normalized";
|
||||
cache.UpsertAsync(change.Document, new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
|
||||
}
|
||||
});
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul:in-place");
|
||||
var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>();
|
||||
using var subscriber = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text)));
|
||||
var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle<NetworkSchemaNote>(key), Note("authoritative"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
await database.ApplyPutAsync(message);
|
||||
|
||||
// The observer's nested write is delivered depth-first, inside the delivery of the change that caused it, so
|
||||
// the database may publish the two in either order; what is pinned is that each carries its own context.
|
||||
Assert.That(seen, Is.EquivalentTo(new (CultNetDatabaseChangeKind, string?)[]
|
||||
{
|
||||
(CultNetDatabaseChangeKind.Predicted, null),
|
||||
(CultNetDatabaseChangeKind.Reconciled, "predicted"),
|
||||
(CultNetDatabaseChangeKind.Updated, "normalized")
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
429
tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs
Normal file
429
tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using GameCult.Caching.MessagePack;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
// The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the
|
||||
// mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every
|
||||
// other change in the hold publishes from the cache.
|
||||
public sealed class CultNetPublishesCommitsTests
|
||||
{
|
||||
private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) =>
|
||||
wire.Changes.Where(change => KeyOf(change) == key).ToArray();
|
||||
|
||||
private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey;
|
||||
|
||||
private sealed class Wire : IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cancellation = new();
|
||||
private readonly Thread _thread;
|
||||
private readonly RudpCultNetSchemaServer _server;
|
||||
private readonly CultNetDatabaseSubscriptionServer _subscriptions;
|
||||
private readonly ICultNetSchemaClient _client;
|
||||
public ConcurrentQueue<CultNetDatabaseChangeRawMessage> Changes { get; } = new();
|
||||
public int Snapshots;
|
||||
public ConcurrentQueue<CultNetErrorMessage> Errors { get; } = new();
|
||||
|
||||
private readonly CultNetDatabase _database;
|
||||
|
||||
public Wire(CultNetDatabase database)
|
||||
{
|
||||
_database = database;
|
||||
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
socket.ReceiveTimeout = 20;
|
||||
_server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket });
|
||||
_subscriptions = new CultNetDatabaseSubscriptionServer(_server, database);
|
||||
_thread = new Thread(() =>
|
||||
{
|
||||
while (!_cancellation.IsCancellationRequested)
|
||||
{
|
||||
_ = _server.PollOnceAsync().GetAwaiter().GetResult();
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}) { IsBackground = true };
|
||||
_thread.Start();
|
||||
_client = CultNetSchemaClients.CreateRudp("publish-client");
|
||||
_client.OnCultNet<CultNetDatabaseChangeRawMessage>(Changes.Enqueue);
|
||||
_client.OnCultNet<CultNetErrorMessage>(Errors.Enqueue);
|
||||
_client.OnCultNet<CultNetSnapshotResponseRawMessage>(_ => Interlocked.Increment(ref Snapshots));
|
||||
_client.Connect("127.0.0.1", _server.LocalEndPoint.Port);
|
||||
}
|
||||
|
||||
public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes)
|
||||
{
|
||||
await Until(() => _client.Connected);
|
||||
_client.SendCultNet(new CultNetDatabaseSubscribeMessage
|
||||
{
|
||||
MessageId = "subscribe",
|
||||
SubscriptionId = "all",
|
||||
SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(),
|
||||
IncludeSnapshot = includeSnapshot
|
||||
});
|
||||
await Until(() => Snapshots > 0 || !Errors.IsEmpty);
|
||||
Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cancellation.Cancel();
|
||||
_thread.Join();
|
||||
_client.Dispose();
|
||||
_subscriptions.Dispose();
|
||||
_server.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task Until(Func<bool> condition)
|
||||
{
|
||||
var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTimeOffset.UtcNow > deadline)
|
||||
Assert.Fail("Condition was not satisfied before timeout.");
|
||||
await Task.Delay(5);
|
||||
}
|
||||
}
|
||||
|
||||
// One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance
|
||||
// to arrive twice.
|
||||
private static async Task SettleAsync(CultNetDatabase database, Wire wire)
|
||||
{
|
||||
var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N");
|
||||
await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" });
|
||||
await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel));
|
||||
}
|
||||
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
|
||||
[Test]
|
||||
public async Task ACacheCommitAndUpsertReachASubscriberOnce()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:commit");
|
||||
|
||||
cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle<NetworkSchemaNote>(key)));
|
||||
await Until(() => Of(wire, key.Value).Length >= 1);
|
||||
cache.Commit(batch => batch.Remove(key));
|
||||
await Until(() => Of(wire, key.Value).Length >= 2);
|
||||
await cache.UpsertAsync(Note("upserted"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADatabasePutAndDeleteStillPublishExactlyOnce()
|
||||
{
|
||||
var database = new CultNetDatabase(new CultCache());
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:put");
|
||||
|
||||
await database.PutAsync(key, Note("put"));
|
||||
await database.DeleteAsync<NetworkSchemaNote>(key);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:replicated");
|
||||
var message = database.Documents.CreateRawDocumentPutMessage(
|
||||
"put-replicated", new CultRecordHandle<NetworkSchemaNote>(key), Note("remote"));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
|
||||
await database.ApplyPutAsync(message);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALoadRemovalCarriesItsKey()
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc");
|
||||
try
|
||||
{
|
||||
var key = new CultRecordKey("tests:publish:loaded");
|
||||
var writer = new CultCache();
|
||||
writer.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
await writer.UpsertAsync(Note("there"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
writer.FlushAllBackingStores();
|
||||
|
||||
var cache = new CultCache();
|
||||
cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
var database = new CultNetDatabase(cache);
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key), Is.Not.Null);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(true, typeof(NetworkSchemaNote));
|
||||
|
||||
writer.Remove(key);
|
||||
writer.FlushAllBackingStores();
|
||||
await cache.PullAllBackingStoresAsync();
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" }));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(path)) File.Delete(path);
|
||||
if (File.Exists(path + ".lock")) File.Delete(path + ".lock");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big"),
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Power), 9)
|
||||
});
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(VariantWireFixture));
|
||||
var errorsBefore = wire.Errors.Count;
|
||||
|
||||
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
await Until(() => wire.Errors.Count > errorsBefore);
|
||||
await Until(() => Of(wire, baseKey.Value).Length >= 1);
|
||||
|
||||
Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value),
|
||||
"the re-resolved dependent is published, then refused (Q6) by key");
|
||||
Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1));
|
||||
Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record");
|
||||
}
|
||||
|
||||
// An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own.
|
||||
[Test]
|
||||
public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
using var wire = new Wire(database);
|
||||
await wire.SubscribeAsync(false, typeof(NetworkSchemaNote));
|
||||
var key = new CultRecordKey("tests:publish:normalized");
|
||||
using var normalizer = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "raw")
|
||||
cache.UpsertAsync(Note("normalized"), new CultRecordHandle<NetworkSchemaNote>(change.Key)).GetAwaiter().GetResult();
|
||||
});
|
||||
|
||||
await database.PutAsync(key, Note("raw"));
|
||||
await Until(() => Of(wire, key.Value).Length >= 2);
|
||||
await SettleAsync(database, wire);
|
||||
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(key)!.Text, Is.EqualTo("normalized"));
|
||||
Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2));
|
||||
}
|
||||
|
||||
private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache()
|
||||
{
|
||||
var registry = CultDocumentRegistry.Shared;
|
||||
var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
|
||||
var documents = new CultNetDocumentRegistry(registry)
|
||||
.Register(CultNetDocumentBinding.ForDocument<VariantWireFixture>(registry));
|
||||
return (cache, documents);
|
||||
}
|
||||
|
||||
private static System.Collections.Generic.List<string> Record(CultNetDatabase database, Func<object, string> describe)
|
||||
{
|
||||
var seen = new System.Collections.Generic.List<string>();
|
||||
database.WatchAllChanges().Subscribe(change => seen.Add(describe(change)));
|
||||
return seen;
|
||||
}
|
||||
|
||||
private static string KeyOfChange(object change) => ((dynamic)change).Key.Value;
|
||||
|
||||
[Test]
|
||||
public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder()
|
||||
{
|
||||
var (cache, documents) = VariantCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents });
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await cache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
cache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
|
||||
});
|
||||
var published = Record(database, KeyOfChange);
|
||||
var logged = database.GetMutationLog(database.Shards[0].ShardId).Count;
|
||||
|
||||
await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value),
|
||||
Is.EqualTo(new[] { baseKey.Value, variantKey.Value }));
|
||||
}
|
||||
|
||||
private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) =>
|
||||
new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = documents,
|
||||
Shards =
|
||||
[
|
||||
new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId])
|
||||
]
|
||||
});
|
||||
|
||||
[Test]
|
||||
public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally()
|
||||
{
|
||||
var (sourceCache, sourceDocuments) = VariantCache();
|
||||
var (replicaCache, replicaDocuments) = VariantCache();
|
||||
var schemaId = sourceCache.Registry.GetRequired<VariantWireFixture>().SchemaId;
|
||||
var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions
|
||||
{
|
||||
DocumentRegistry = sourceDocuments,
|
||||
Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])]
|
||||
});
|
||||
var replica = Replica(replicaCache, replicaDocuments, schemaId);
|
||||
var baseKey = new CultRecordKey("publish-variant-base");
|
||||
var variantKey = new CultRecordKey("publish-variant-big");
|
||||
await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle<VariantWireFixture>(baseKey));
|
||||
await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[]
|
||||
{
|
||||
replicaCache.Override<VariantWireFixture>(nameof(VariantWireFixture.Name), "big")
|
||||
});
|
||||
await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 });
|
||||
|
||||
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = "replica-shard",
|
||||
ShardEpoch = 3,
|
||||
Entries = source.GetMutationLogMessages("replica-shard").ToArray()
|
||||
});
|
||||
|
||||
Assert.That(replicaCache.Get<VariantWireFixture>(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved");
|
||||
Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }),
|
||||
"the replica log holds the primary's entry and mints none of its own");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
var replica = Replica(cache, null, schemaId);
|
||||
var published = Record(replica, KeyOfChange);
|
||||
|
||||
await cache.UpsertAsync(Note("local"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:local")));
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" }));
|
||||
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
const int writers = 4, perWriter = 5000;
|
||||
|
||||
Parallel.For(0, writers, writer =>
|
||||
{
|
||||
for (var i = 0; i < perWriter; i++)
|
||||
cache.UpsertAsync(Note("n"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey($"tests:publish:c:{writer}:{i}")))
|
||||
.GetAwaiter().GetResult();
|
||||
});
|
||||
|
||||
var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray();
|
||||
Assert.That(sequences, Has.Length.EqualTo(writers * perWriter));
|
||||
Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var key = new CultRecordKey("tests:publish:after-delete");
|
||||
await database.PutAsync(key, Note("one"));
|
||||
await database.DeleteAsync<NetworkSchemaNote>(key);
|
||||
var published = Record(database, KeyOfChange);
|
||||
|
||||
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(key));
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { key.Value }));
|
||||
Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind),
|
||||
Is.EqualTo(new[]
|
||||
{
|
||||
CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added
|
||||
}));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var schemaId = cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
var replica = Replica(cache, null, schemaId);
|
||||
var stale = new CultRecordKey("tests:publish:stale");
|
||||
var fresh = new CultRecordKey("tests:publish:fresh");
|
||||
await cache.UpsertAsync(Note("stale"), new CultRecordHandle<NetworkSchemaNote>(stale));
|
||||
var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value);
|
||||
var put = replica.Documents.CreateRawDocumentPutMessage(
|
||||
"snapshot-fresh", new CultRecordHandle<NetworkSchemaNote>(fresh), Note("fresh"));
|
||||
|
||||
await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage
|
||||
{
|
||||
ShardId = "replica-shard",
|
||||
ShardEpoch = 3,
|
||||
Documents = [put.Document!],
|
||||
ShardLogSequence = 5
|
||||
});
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value }));
|
||||
Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADisposedDatabaseNoLongerObservesTheCache()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var published = Record(database, KeyOfChange);
|
||||
database.Dispose();
|
||||
|
||||
await cache.UpsertAsync(Note("after"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("tests:publish:disposed")));
|
||||
|
||||
Assert.That(published, Is.Empty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -195,6 +195,17 @@ namespace GameCult.Networking.Tests
|
|||
AssertErrorMatchesSchema(CultNetErrorMessage.ForReferenceOutsideTarget(ex));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ErrorMessage_UnownedSchemaRealBytes_MatchesItsSchema()
|
||||
{
|
||||
var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")));
|
||||
|
||||
Assert.That(message.Code, Is.EqualTo("unowned_schema"));
|
||||
Assert.That(message.Details!.Field, Is.EqualTo("schemaId"));
|
||||
Assert.That(message.Details.Value, Is.EqualTo("some.schema.v1"));
|
||||
AssertErrorMatchesSchema(message);
|
||||
}
|
||||
|
||||
private static void AssertErrorMatchesSchema(CultNetErrorMessage message)
|
||||
{
|
||||
var validator = new MiniJsonSchemaValidator(SchemaDir());
|
||||
|
|
|
|||
|
|
@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests
|
|||
Assert.That(cache.Get(batched), Is.Null);
|
||||
}), Is.True);
|
||||
|
||||
Assert.That(published, Is.EqualTo(new[] { put }));
|
||||
Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it");
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(batched)?.Text, Is.EqualTo("batched"));
|
||||
var reopened = new CultCache();
|
||||
reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
|
|
@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests
|
|||
|
||||
var cache = new CultCache();
|
||||
var delivering = new ManualResetEventSlim();
|
||||
cache.OnUpdate += (_, _) => delivering.Set();
|
||||
// Subscribed before the database, so it runs before the database's own observer of the same change.
|
||||
using var loadDelivering = cache.Watch<object>().Subscribe(_ => delivering.Set());
|
||||
var database = new CultNetDatabase(cache);
|
||||
var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions
|
||||
{
|
||||
|
|
@ -4211,13 +4212,13 @@ namespace GameCult.Networking.Tests
|
|||
Schema = "tests.networking_note.v1",
|
||||
Text = "two"
|
||||
});
|
||||
// Outside the shard's key prefix: must reach the non-shard snapshot but never the
|
||||
// Outside the shard's key prefix (written to the cache directly: the database refuses a write no shard owns): must reach the non-shard snapshot but never the
|
||||
// shard-scoped one, so an unfiltered request still proves shard membership is enforced.
|
||||
await database.PutAsync(new CultRecordKey("other-note:outside-shard"), new NetworkSchemaNote
|
||||
await cache.UpsertAsync(new NetworkSchemaNote
|
||||
{
|
||||
Schema = "tests.networking_note.v1",
|
||||
Text = "outside"
|
||||
});
|
||||
}, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("other-note:outside-shard")));
|
||||
|
||||
var nonShard = registry.CreateRawSnapshotResponse(cache, "snapshot-nonshard-unfiltered", filter: null);
|
||||
var shardScoped = database.CreateShardSnapshotResponse(shard, "snapshot-shard-unfiltered", filter: null);
|
||||
|
|
|
|||
304
tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs
Normal file
304
tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
// Soul pass on Cut 4 (scratch; not for merge). Category Finding asserts the invariant Soul expects and is expected to
|
||||
// fail on 13072fc5; Pin asserts a promise that should hold; Measure reports numbers.
|
||||
public sealed class SoulP4ProbeTests
|
||||
{
|
||||
private const string ShardId = "soul-shard";
|
||||
private static readonly CultRecordKey One = new("soul:p4:one");
|
||||
private static readonly CultRecordKey Two = new("soul:p4:two");
|
||||
private static readonly CultRecordKey Three = new("soul:p4:three");
|
||||
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
private static string SchemaId(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
|
||||
private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) =>
|
||||
new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])],
|
||||
MutationLogStore = store
|
||||
});
|
||||
|
||||
private sealed class Store : ICultNetShardMutationLogStore
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly SortedDictionary<long, CultNetShardLogEntryMessage> _entries = new();
|
||||
private long _compacted;
|
||||
public Func<long, bool> RefuseAppend { get; set; } = _ => false;
|
||||
public bool RefuseCompact { get; set; }
|
||||
public int AppendDelayMs { get; set; }
|
||||
public ManualResetEventSlim InAppend { get; } = new();
|
||||
|
||||
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var e = _entries.Values.Where(x => x.Sequence > afterSequence);
|
||||
return (limit.HasValue ? e.Take(limit.Value) : e).ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
public void Append(string shardId, CultNetShardLogEntryMessage entry)
|
||||
{
|
||||
InAppend.Set();
|
||||
if (AppendDelayMs > 0) Thread.Sleep(AppendDelayMs);
|
||||
lock (_gate)
|
||||
{
|
||||
if (RefuseAppend(entry.Sequence)) throw new IOException($"refused {entry.Sequence}");
|
||||
_entries[entry.Sequence] = entry;
|
||||
}
|
||||
}
|
||||
|
||||
public long GetCompactedThrough(string shardId) { lock (_gate) return _compacted; }
|
||||
|
||||
public void CompactThrough(string shardId, long sequence)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (RefuseCompact) throw new IOException("refused compact");
|
||||
_compacted = Math.Max(_compacted, sequence);
|
||||
foreach (var k in _entries.Keys.Where(k => k <= sequence).ToArray()) _entries.Remove(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Peer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable
|
||||
{
|
||||
private readonly Server _server;
|
||||
private readonly CultNetDatabaseServer _dbServer;
|
||||
private readonly CultNetDatabase _db;
|
||||
public Peer(CultCache cache, CultNetDatabase db)
|
||||
{
|
||||
_db = db;
|
||||
_server = new Server(cache, ServerSecurityOptions.Development());
|
||||
_dbServer = new CultNetDatabaseServer(_server, db);
|
||||
}
|
||||
public CultNetShardLogResponseMessage Log(long after) => _dbServer.CreateShardLogResponse(new CultNetShardLogRequestMessage
|
||||
{ MessageId = "pull", ShardId = ShardId, ShardEpoch = 1, AfterSequence = after });
|
||||
public Task<CultNetShardLogResponseMessage> FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => Task.FromResult(Log(afterSequence));
|
||||
public Task<CultNetSnapshotResponseRawMessage> FetchAsync(CultNetShardDescriptor shard) => Task.FromResult(_db.CreateShardSnapshotResponse(_db.Shards[0], "snap"));
|
||||
public void Dispose() { _dbServer.Dispose(); _server.Dispose(); }
|
||||
}
|
||||
|
||||
private static int StashCount(CultNetDatabase db)
|
||||
{
|
||||
var field = typeof(CultNetDatabase).GetField("_stash", BindingFlags.NonPublic | BindingFlags.Instance)!;
|
||||
var stash = (System.Collections.ICollection)field.GetValue(db)!;
|
||||
lock (typeof(CultNetDatabase).GetField("_logGate", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(db)!)
|
||||
return stash.Count;
|
||||
}
|
||||
|
||||
// F1: the recorded residual is "tail reuse". A hole in the middle of the log after a double refusal and a restart
|
||||
// strands a replica behind it for good: the restarted primary has no floor, the store serves S+1 after S-1.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task F1_MidLogHoleAfterDoubleRefusalAndRestartStrandsAReplica()
|
||||
{
|
||||
var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true };
|
||||
var cache = new CultCache();
|
||||
var primary = Db(cache, true, store);
|
||||
var replicaCache = new CultCache();
|
||||
var replica = Db(replicaCache, false, null);
|
||||
await primary.PutAsync(One, Note("one"));
|
||||
using (var p0 = new Peer(cache, primary))
|
||||
using (var r0 = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = p0, SnapshotFetcher = p0 }))
|
||||
await r0.PullOnceAsync(ShardId);
|
||||
try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { }
|
||||
store.RefuseCompact = false;
|
||||
await primary.PutAsync(Three, Note("three"));
|
||||
primary.Dispose();
|
||||
|
||||
var restarted = Db(cache, true, store);
|
||||
using var peer = new Peer(cache, restarted);
|
||||
using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer });
|
||||
var errors = new List<string>();
|
||||
for (var attempt = 0; attempt < 3; attempt++)
|
||||
{
|
||||
try { await replicator.PullOnceAsync(ShardId); }
|
||||
catch (Exception e) { errors.Add(e.GetType().Name + ": " + e.Message); }
|
||||
}
|
||||
|
||||
TestContext.Out.WriteLine($"store={string.Join(",", store.Read(ShardId).Select(e => e.Sequence))} floor={restarted.GetCompactedMutationLogSequence(ShardId)} errors=[{string.Join(" | ", errors)}]");
|
||||
Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get<NetworkSchemaNote>(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" }));
|
||||
}
|
||||
|
||||
// F2: Q-P5 A says the writer gets a typed CultNetShardLogException. A commit of two changes on a refusing store
|
||||
// hands the writer an AggregateException; a writer's catch (CultNetShardLogException) misses it.
|
||||
[Test, Category("SoulFinding")]
|
||||
public void F2_ABatchOnARefusingStoreReachesTheWriterAsTheTypedException()
|
||||
{
|
||||
var store = new Store { RefuseAppend = _ => true };
|
||||
var cache = new CultCache();
|
||||
_ = Db(cache, true, store);
|
||||
Exception? caught = null;
|
||||
try
|
||||
{
|
||||
cache.Commit(batch =>
|
||||
{
|
||||
batch.Upsert(Note("one"), new CultRecordHandle<NetworkSchemaNote>(One));
|
||||
batch.Upsert(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
|
||||
});
|
||||
}
|
||||
catch (Exception e) { caught = e; }
|
||||
TestContext.Out.WriteLine($"writer caught {caught?.GetType().FullName}");
|
||||
Assert.That(caught, Is.InstanceOf<CultNetShardLogException>());
|
||||
}
|
||||
|
||||
// F3: main publishes PutPredictedAsync as Predicted and records the prediction whatever instance it is given. The
|
||||
// shape rule refuses a door whose instance is already the cached one, so a mutate-then-predict is published as
|
||||
// Updated, is logged on a primary, and its reconciliation is published as Updated.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task F3_APredictionOfTheCachedInstanceIsPublishedAsAPrediction()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var db = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var shard = db.Shards[0];
|
||||
var kinds = new List<CultNetDatabaseChangeKind>();
|
||||
db.Watch<NetworkSchemaNote>().Subscribe(c => kinds.Add(c.Kind));
|
||||
await db.PutPredictedAsync(One, Note("p1"));
|
||||
var held = cache.Get<NetworkSchemaNote>(One)!;
|
||||
held.Text = "p2";
|
||||
await db.PutPredictedAsync(One, held);
|
||||
var auth = db.Documents.CreateRawDocumentPutMessage("a", new CultRecordHandle<NetworkSchemaNote>(One), Note("auth"));
|
||||
auth.ShardId = shard.ShardId;
|
||||
auth.ShardEpoch = shard.Epoch;
|
||||
await db.ApplyPutAsync(auth);
|
||||
TestContext.Out.WriteLine($"kinds=[{string.Join(",", kinds)}] log=[{string.Join(",", db.GetMutationLog(shard.ShardId).Select(e => e.Kind))}]");
|
||||
Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled }));
|
||||
Assert.That(db.GetMutationLog(shard.ShardId).Select(e => e.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated }));
|
||||
}
|
||||
|
||||
// F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an
|
||||
// unrelated key waits for the whole append.
|
||||
[Test, Category("SoulFinding"), Ignore("Accepted follow-up F4: a log append runs under the cache gate, so readers wait; documented on ICultNetShardMutationLogStore.")]
|
||||
public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend()
|
||||
{
|
||||
var store = new Store { AppendDelayMs = 400 };
|
||||
var cache = new CultCache();
|
||||
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
|
||||
var db = Db(cache, true, store);
|
||||
var writer = Task.Run(() => cache.UpsertAsync(Note("one"), new CultRecordHandle<NetworkSchemaNote>(One)).GetAwaiter().GetResult());
|
||||
Assert.That(store.InAppend.Wait(5000), Is.True);
|
||||
var clock = Stopwatch.StartNew();
|
||||
_ = cache.Get<NetworkSchemaNote>(Two);
|
||||
var waited = clock.Elapsed.TotalMilliseconds;
|
||||
await writer;
|
||||
TestContext.Out.WriteLine($"reader of an unrelated key waited {waited:0.0} ms for a 400 ms append");
|
||||
Assert.That(waited, Is.LessThan(100));
|
||||
}
|
||||
|
||||
// P1: the constructor registers the observer before the journal, so a change stashed by the journal always finds
|
||||
// its publisher, even while another thread writes the cache during construction.
|
||||
[Test, Category("SoulPin")]
|
||||
public void P1_ConstructionUnderConcurrentWritesOrphansNoStash()
|
||||
{
|
||||
var orphans = 0;
|
||||
for (var round = 0; round < 200; round++)
|
||||
{
|
||||
var cache = new CultCache();
|
||||
using var stop = new CancellationTokenSource();
|
||||
var writer = Task.Run(() =>
|
||||
{
|
||||
var i = 0;
|
||||
while (!stop.IsCancellationRequested)
|
||||
cache.UpsertAsync(Note("w" + i++), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:race:" + (i % 4)))).GetAwaiter().GetResult();
|
||||
});
|
||||
Thread.SpinWait(200 * (round % 7));
|
||||
var db = Db(cache, true, null);
|
||||
Thread.SpinWait(2000);
|
||||
stop.Cancel();
|
||||
writer.GetAwaiter().GetResult();
|
||||
orphans += StashCount(db);
|
||||
db.Dispose();
|
||||
}
|
||||
|
||||
Assert.That(orphans, Is.EqualTo(0), "no stash entry outlives its admission");
|
||||
}
|
||||
|
||||
// P2: every stash entry is removed by its publication.
|
||||
[Test, Category("SoulPin")]
|
||||
public async Task P2_TheStashDrainsAfterEveryWrite()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var db = Db(cache, true, new Store());
|
||||
await db.PutAsync(One, Note("one"));
|
||||
await cache.UpsertAsync(Note("two"), new CultRecordHandle<NetworkSchemaNote>(Two));
|
||||
cache.Commit(batch => batch.Upsert(Note("three"), new CultRecordHandle<NetworkSchemaNote>(Three)));
|
||||
await db.DeleteAsync<NetworkSchemaNote>(One);
|
||||
Assert.That(StashCount(db), Is.EqualTo(0));
|
||||
}
|
||||
|
||||
// P3: a replica never applies an entry past a hole. The primary burns 2 with compaction refused, then logs 3; a
|
||||
// replica at 1 is answered compacted_history and resyncs, never applying 3 on top of 1.
|
||||
[Test, Category("SoulPin")]
|
||||
public async Task P3_AReplicaAtTheHoleResyncsRatherThanSkippingIt()
|
||||
{
|
||||
var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true };
|
||||
var cache = new CultCache();
|
||||
var primary = Db(cache, true, store);
|
||||
using var peer = new Peer(cache, primary);
|
||||
var replicaCache = new CultCache();
|
||||
var replica = Db(replicaCache, false, null);
|
||||
using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer });
|
||||
await primary.PutAsync(One, Note("one"));
|
||||
await replicator.PullOnceAsync(ShardId);
|
||||
try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { }
|
||||
await primary.PutAsync(Three, Note("three"));
|
||||
Assert.That(peer.Log(1).Reason, Is.EqualTo("compacted_history"));
|
||||
var applied = await replicator.PullOnceAsync(ShardId);
|
||||
Assert.That(applied, Is.EqualTo(3));
|
||||
Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get<NetworkSchemaNote>(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" }));
|
||||
}
|
||||
|
||||
[Test, Category("SoulMeasure"), Explicit("Measurement, no assertion; writes a log file.")]
|
||||
public async Task M1_BareWriteLatencyWithTheFileLogStore()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
var cache = new CultCache();
|
||||
_ = Db(cache, true, new CultNetFileShardMutationLogStore(root));
|
||||
var times = new List<double>();
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
var clock = Stopwatch.StartNew();
|
||||
await cache.UpsertAsync(Note("n" + i), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:lat:" + i)));
|
||||
times.Add(clock.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
|
||||
TestContext.Out.WriteLine($"file store bare write ms: first100 avg {times.Take(100).Average():0.000}, 400-500 avg {times.Skip(400).Take(100).Average():0.000}, last100 avg {times.Skip(900).Average():0.000}, max {times.Max():0.0}");
|
||||
var bare = new CultCache();
|
||||
var t2 = new List<double>();
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
var clock = Stopwatch.StartNew();
|
||||
await bare.UpsertAsync(Note("n" + i), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul:p4:lat:" + i)));
|
||||
t2.Add(clock.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
|
||||
TestContext.Out.WriteLine($"no database bare write ms: last100 avg {t2.Skip(900).Average():0.000}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(root)) Directory.Delete(root, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
142
tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs
Normal file
142
tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
// Soul pass 4c probes. They guard: a committed put whose shard log failed is not answered as a refusal (Q1); reads and watch agree
|
||||
// on what a database serves (Q2); a replica log apply leaves out a row its shard does not own (Q3); the unowned refusal is not a
|
||||
// shard-authority exception (Q4); two databases over one cache aggregate their log failures (Q5).
|
||||
public sealed class SoulP4cProbeTests
|
||||
{
|
||||
private const string ShardId = "soulc-shard";
|
||||
private static readonly CultRecordKey One = new("soulc:one");
|
||||
private static readonly CultRecordKey Unowned = new("soulc:unowned");
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
private static string SchemaId(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
|
||||
private sealed class RefusingStore : ICultNetShardMutationLogStore
|
||||
{
|
||||
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null) => Array.Empty<CultNetShardLogEntryMessage>();
|
||||
public void Append(string shardId, CultNetShardLogEntryMessage entry) => throw new IOException("refused " + entry.Sequence);
|
||||
public long GetCompactedThrough(string shardId) => 0;
|
||||
public void CompactThrough(string shardId, long sequence) { }
|
||||
}
|
||||
|
||||
private sealed class Logger : GameCult.Logging.ILogger
|
||||
{
|
||||
public List<string> Errors { get; } = new();
|
||||
public void LogInfo(string message) { }
|
||||
public void LogWarning(string message) { }
|
||||
public void LogError(string message) => Errors.Add(message);
|
||||
public void LogDebug(string message) { }
|
||||
}
|
||||
|
||||
private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) =>
|
||||
new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])],
|
||||
MutationLogStore = store
|
||||
});
|
||||
|
||||
// Q1: the server's own remote-put path is a writer. A put that committed and could not be logged is answered as a
|
||||
// failure: the peer is told its write failed, while the cache holds it and it was published.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q1_ServerAnswersACommittedButUnloggedRemotePutAsAFailure()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = Db(cache, primary: true, new RefusingStore());
|
||||
using var server = new Server(cache, ServerSecurityOptions.Development());
|
||||
var logger = new Logger();
|
||||
server.Logger = logger;
|
||||
using var databaseServer = new CultNetDatabaseServer(server, database);
|
||||
var handler = (Func<CultNetDocumentPutRawMessage, CultNetServerPeer, Task>)typeof(CultNetDatabaseServer)
|
||||
.GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
|
||||
.GetValue(databaseServer)!;
|
||||
var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer));
|
||||
var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<NetworkSchemaNote>(One), Note("wire"));
|
||||
message.ShardId = ShardId;
|
||||
message.ShardEpoch = 1;
|
||||
|
||||
try { await handler(message, peer); } catch (Exception) { }
|
||||
|
||||
TestContext.WriteLine("errors: " + string.Join(" | ", logger.Errors));
|
||||
Assert.That(cache.Get<NetworkSchemaNote>(One)?.Text, Is.EqualTo("wire"), "the put committed");
|
||||
Assert.That(logger.Errors, Has.None.Contains("raw put failed"), "a committed put must not be reported to the peer as a failed one");
|
||||
}
|
||||
|
||||
// Q2: reads serve a cached row no shard owns, the database watch never publishes its changes: a view built from
|
||||
// GetAll + Watch (CultMesh Collection, the subscription server's snapshot + WatchAllChanges) goes stale.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q2_ReadsServeAnUnownedRowWhoseChangesTheWatchNeverPublishes()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = Db(cache, primary: true, store: null);
|
||||
var seen = new List<string>();
|
||||
database.Watch<MeshQuickstartNote>().Subscribe(change => seen.Add(change.Document?.Body ?? "<removed>"));
|
||||
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v1" }, new CultRecordHandle<MeshQuickstartNote>(Unowned));
|
||||
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v2" }, new CultRecordHandle<MeshQuickstartNote>(Unowned));
|
||||
|
||||
var read = database.GetAll<MeshQuickstartNote>().Select(note => note.Body).ToArray();
|
||||
TestContext.WriteLine($"GetAll={string.Join(",", read)} watch={string.Join(",", seen)}");
|
||||
Assert.That(read.Length == 0 || seen.Count > 0, Is.True, "a row the database reads out must be one whose changes it publishes");
|
||||
}
|
||||
|
||||
// Q3: a replica's log apply door writes a row its shard does not own. The snapshot apply path filters by
|
||||
// ShardMatchesRawDocument; the log path does not.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q3_AReplicaLogApplyWritesARowItsShardDoesNotOwn()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var replica = Db(cache, primary: false, store: null);
|
||||
var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(Unowned), new MeshQuickstartNote { NoteId = "r", Body = "replicated" });
|
||||
put.ShardId = ShardId;
|
||||
put.ShardEpoch = 1;
|
||||
Exception? thrown = null;
|
||||
try
|
||||
{
|
||||
await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
MessageId = "log",
|
||||
ShardId = ShardId,
|
||||
ShardEpoch = 1,
|
||||
Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = DateTimeOffset.UtcNow.ToString("O"), ChangeKind = "added", Put = put }]
|
||||
});
|
||||
}
|
||||
catch (Exception exception) { thrown = exception; }
|
||||
|
||||
TestContext.WriteLine($"thrown={thrown?.GetType().Name}: {thrown?.Message}; cached={cache.Get<MeshQuickstartNote>(Unowned)?.Body}; log={replica.GetMutationLog(ShardId).Count}");
|
||||
Assert.That(cache.Get<MeshQuickstartNote>(Unowned), Is.Null, "no shard owns it here, so no path writes it");
|
||||
}
|
||||
|
||||
// Q4 (pin): the unowned refusal is not a shard-authority exception, so the server's routing path does not see it.
|
||||
[Test, Category("SoulPin")]
|
||||
public void Q4_TheUnownedRefusalIsNotAShardAuthorityException()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = Db(cache, primary: true, store: null);
|
||||
var put = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "w" });
|
||||
var refusal = Assert.ThrowsAsync<CultNetUnownedSchemaException>(async () => await database.ApplyPutAsync(put))!;
|
||||
Assert.That(refusal, Is.Not.InstanceOf<CultNetShardAuthorityException>());
|
||||
}
|
||||
|
||||
// Q5 (pin): two databases over one cache, both refusing: a writer's catch (CultNetShardLogException) misses.
|
||||
[Test, Category("SoulPin")]
|
||||
public void Q5_TwoDatabasesOnOneCacheAggregateTheirLogFailures()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
_ = Db(cache, primary: true, new RefusingStore());
|
||||
_ = Db(cache, primary: true, new RefusingStore());
|
||||
var failure = Assert.Throws<AggregateException>(() => cache.Commit(batch =>
|
||||
batch.Upsert(Note("x"), new CultRecordHandle<NetworkSchemaNote>(One))))!;
|
||||
Assert.That(failure.InnerExceptions.OfType<CultNetShardLogException>().Count(), Is.EqualTo(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
201
tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs
Normal file
201
tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
// Soul pass 4d probes. They guard the replica log relay (a skipped entry is relayed as the primary sent it, in memory and on
|
||||
// disk), key-prefixed shard reads, and measure two costs; the measurements are explicit.
|
||||
public sealed class SoulP4dProbeTests
|
||||
{
|
||||
private const string ShardId = "soupd-shard";
|
||||
private static readonly string Now = DateTimeOffset.UtcNow.ToString("O");
|
||||
|
||||
private static string NoteSchema(CultCache cache) => cache.Registry.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
private static string MeshSchema(CultCache cache) => cache.Registry.GetRequired<MeshQuickstartNote>().SchemaId;
|
||||
|
||||
private static CultNetDatabase Db(CultCache cache, bool primary, string[] schemas, string? keyPrefix = null, ICultNetShardMutationLogStore? store = null) =>
|
||||
new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: schemas, keyPrefix: keyPrefix)],
|
||||
MutationLogStore = store
|
||||
});
|
||||
|
||||
private static CultNetShardLogResponseMessage UnownedPutLog(CultNetDatabase anyDb, string key)
|
||||
{
|
||||
var put = anyDb.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey(key)), new MeshQuickstartNote { NoteId = "p", Body = "primary-put" });
|
||||
put.ShardId = ShardId;
|
||||
put.ShardEpoch = 1;
|
||||
return new CultNetShardLogResponseMessage
|
||||
{
|
||||
MessageId = "log",
|
||||
ShardId = ShardId,
|
||||
ShardEpoch = 1,
|
||||
Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = Now, ChangeKind = "added", Put = put }]
|
||||
};
|
||||
}
|
||||
|
||||
// Q3a: a replica (no store) that skips an unowned put relays it to a chained replica as a delete.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q3a_MemoryLogRelaysASkippedPutAsARemoval()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var replica = Db(cache, primary: false, [NoteSchema(cache)]);
|
||||
await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k"));
|
||||
var relayed = replica.GetMutationLogMessages(ShardId);
|
||||
TestContext.Out.WriteLine($"memory relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null} delete={relayed[0].Delete != null}");
|
||||
Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"), "the relayed entry keeps the primary's kind");
|
||||
}
|
||||
|
||||
// Q3b: the same replica with a durable store relays the primary's put (disk and memory disagree).
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q3b_StoreBackedLogRelaysTheSkippedPutAsAPut()
|
||||
{
|
||||
var dir = Path.Combine(Path.GetTempPath(), "soulp4d-" + Guid.NewGuid().ToString("N"));
|
||||
var cache = new CultCache();
|
||||
var replica = Db(cache, primary: false, [NoteSchema(cache)], store: new CultNetFileShardMutationLogStore(dir));
|
||||
await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k"));
|
||||
var relayed = replica.GetMutationLogMessages(ShardId);
|
||||
var memory = replica.GetMutationLog(ShardId);
|
||||
TestContext.Out.WriteLine($"store relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null}; memory entry kind={memory[0].Kind} doc={(memory[0].Document == null ? "null" : "set")}");
|
||||
Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"));
|
||||
}
|
||||
|
||||
// Q3c: a chained replica that does own the row (config drift) and holds it deletes it on the relayed entry.
|
||||
[Test, Category("SoulFinding")]
|
||||
public async Task Q3c_AChainedReplicaThatOwnsTheRowDeletesWhatThePrimaryPut()
|
||||
{
|
||||
var cacheA = new CultCache();
|
||||
var a = Db(cacheA, primary: false, [NoteSchema(cacheA)]);
|
||||
await a.ApplyShardLogResponseAsync(UnownedPutLog(a, "relay:k"));
|
||||
|
||||
var cacheC = new CultCache();
|
||||
var c = Db(cacheC, primary: false, [NoteSchema(cacheC), MeshSchema(cacheC)]);
|
||||
await cacheC.UpsertAsync(new MeshQuickstartNote { NoteId = "p", Body = "earlier" }, new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey("relay:k")));
|
||||
await c.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
MessageId = "chain",
|
||||
ShardId = ShardId,
|
||||
ShardEpoch = 1,
|
||||
Entries = a.GetMutationLogMessages(ShardId).ToArray()
|
||||
});
|
||||
var row = cacheC.Get<MeshQuickstartNote>(new CultRecordKey("relay:k"));
|
||||
TestContext.Out.WriteLine($"chained row after relay: {row?.Body ?? "<deleted>"}");
|
||||
Assert.That(row, Is.Not.Null, "the primary put this row; a relay must not delete it");
|
||||
}
|
||||
|
||||
// A skipped entry is relayed exactly as the primary sent it, whether the replica keeps its log in memory or on disk.
|
||||
[Test]
|
||||
public async Task SkippedEntriesAreRelayedIdenticallyFromMemoryAndFromDisk()
|
||||
{
|
||||
var dir = Path.Combine(Path.GetTempPath(), "soulp4d-parity-" + Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
var memoryCache = new CultCache();
|
||||
var inMemory = Db(memoryCache, primary: false, [NoteSchema(memoryCache)]);
|
||||
var diskCache = new CultCache();
|
||||
var onDisk = Db(diskCache, primary: false, [NoteSchema(diskCache)], store: new CultNetFileShardMutationLogStore(dir));
|
||||
var log = UnownedPutLog(inMemory, "relay:k");
|
||||
var removal = new CultNetShardLogEntryMessage
|
||||
{
|
||||
Sequence = 2,
|
||||
CommittedAt = Now,
|
||||
ChangeKind = "removed",
|
||||
Delete = new CultNetDocumentDeleteMessage
|
||||
{
|
||||
MessageId = "d",
|
||||
SchemaId = memoryCache.Registry.GetRequired<MeshQuickstartNote>().SchemaId,
|
||||
RecordKey = "relay:k",
|
||||
ShardId = ShardId,
|
||||
ShardEpoch = 1
|
||||
}
|
||||
};
|
||||
log.Entries = [log.Entries[0], removal];
|
||||
|
||||
await inMemory.ApplyShardLogResponseAsync(log);
|
||||
await onDisk.ApplyShardLogResponseAsync(log);
|
||||
|
||||
var memory = inMemory.GetMutationLogMessages(ShardId);
|
||||
var disk = onDisk.GetMutationLogMessages(ShardId);
|
||||
Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(new[] { "added", "removed" }));
|
||||
Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(disk.Select(e => e.ChangeKind)));
|
||||
Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(new[] { true, false }));
|
||||
Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(disk.Select(e => e.Put != null)));
|
||||
Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(new[] { false, true }));
|
||||
Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(disk.Select(e => e.Delete != null)));
|
||||
Assert.That(memory[0].Put!.Document!.RecordKey, Is.EqualTo(disk[0].Put!.Document!.RecordKey));
|
||||
Assert.That(memory[0].Put!.Document!.Payload, Is.EqualTo(disk[0].Put!.Document!.Payload), "the document bytes are the primary's");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(dir)) Directory.Delete(dir, true);
|
||||
}
|
||||
}
|
||||
|
||||
// Q2: key-prefixed shard: the schema is owned, a key outside the prefix is not.
|
||||
[Test, Category("SoulPin")]
|
||||
public async Task Q2_KeyPrefixedReadsServeOnlyTheOwnedKeys()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = Db(cache, primary: true, [NoteSchema(cache)], keyPrefix: "a:");
|
||||
await cache.UpsertAsync(new NetworkSchemaNote { Text = "a" }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("a:1")));
|
||||
await cache.UpsertAsync(new NetworkSchemaNote { Text = "b" }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("b:1")));
|
||||
Assert.That(database.GetAll<NetworkSchemaNote>().Select(n => n.Text), Is.EqualTo(new[] { "a" }));
|
||||
Assert.That(await database.GetAsync<NetworkSchemaNote>(new CultRecordKey("b:1")), Is.Null);
|
||||
Assert.That(await database.GetAsync<NetworkSchemaNote>(new CultRecordKey("a:1")), Is.Not.Null);
|
||||
using var server = new Server(cache, ServerSecurityOptions.Development());
|
||||
using var databaseServer = new CultNetDatabaseServer(server, database);
|
||||
var snapshot = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" });
|
||||
Assert.That(snapshot.Documents.Select(d => d.RecordKey), Is.EqualTo(new[] { "a:1" }));
|
||||
}
|
||||
|
||||
[Test, Category("SoulPin")]
|
||||
public async Task Q2_PrimaryAllServesEveryCachedRow()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { Shards = [CultNetShardDescriptor.PrimaryAll()] });
|
||||
await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "x", Body = "x" }, new CultRecordHandle<MeshQuickstartNote>(new CultRecordKey("any:1")));
|
||||
Assert.That(database.GetAll<object>().Count(), Is.EqualTo(1));
|
||||
Assert.That(database.GetByName<MeshQuickstartNote>("x"), Is.Not.Null);
|
||||
}
|
||||
|
||||
// Q2 cost: GetAll through the database versus the cache, scoped shard (reflection per row).
|
||||
[Test, Category("SoulMeasure"), Explicit("Measurement, no assertion.")]
|
||||
public void Q2_GetAllCost()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = Db(cache, primary: false, [NoteSchema(cache)], keyPrefix: "n:");
|
||||
cache.Commit(batch =>
|
||||
{
|
||||
for (var i = 0; i < 20000; i++)
|
||||
batch.Upsert(new NetworkSchemaNote { Text = "t" + i }, new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("n:" + i)));
|
||||
});
|
||||
_ = cache.GetAll<NetworkSchemaNote>().Count();
|
||||
_ = database.GetAll<NetworkSchemaNote>().Count();
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var r = 0; r < 5; r++) _ = cache.GetAll<NetworkSchemaNote>().Count();
|
||||
var cacheMs = sw.Elapsed.TotalMilliseconds / 5;
|
||||
sw.Restart();
|
||||
for (var r = 0; r < 5; r++) _ = database.GetAll<NetworkSchemaNote>().Count();
|
||||
var dbMs = sw.Elapsed.TotalMilliseconds / 5;
|
||||
TestContext.Out.WriteLine($"GetAll 20000 rows: cache={cacheMs:F2}ms database={dbMs:F2}ms ratio={dbMs / Math.Max(cacheMs, 0.001):F1}x");
|
||||
}
|
||||
|
||||
// Wire: the bytes a peer receives for the unowned refusal.
|
||||
[Test, Category("SoulMeasure"), Explicit("Prints the wire bytes the Rust captured-bytes case pins.")]
|
||||
public void Wire_UnownedSchemaBytes()
|
||||
{
|
||||
var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key")));
|
||||
var bytes = MessagePack.MessagePackSerializer.Serialize(message, CultNetSchemaMessageSerialization.Options);
|
||||
TestContext.Out.WriteLine("HEX=" + Convert.ToHexString(bytes));
|
||||
TestContext.Out.WriteLine("JSON=" + MessagePack.MessagePackSerializer.ConvertToJson(bytes));
|
||||
}
|
||||
}
|
||||
}
|
||||
276
tests/GameCult.Networking.Tests/SoulPub3Tests.cs
Normal file
276
tests/GameCult.Networking.Tests/SoulPub3Tests.cs
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using NUnit.Framework;
|
||||
using R3;
|
||||
using static GameCult.Networking.Tests.NetworkingTests;
|
||||
|
||||
namespace GameCult.Networking.Tests
|
||||
{
|
||||
[CultDocument("tests.soul_throwing_name", "tests.soul_throwing_name.v1")]
|
||||
[MessagePack.MessagePackObject]
|
||||
public sealed class SoulThrowingName
|
||||
{
|
||||
[MessagePack.Key(0)]
|
||||
public string Text { get; set; } = string.Empty;
|
||||
|
||||
[MessagePack.Key(1)]
|
||||
[CultName]
|
||||
public string Name
|
||||
{
|
||||
get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text;
|
||||
set { }
|
||||
}
|
||||
}
|
||||
|
||||
[NonParallelizable]
|
||||
public sealed class SoulPub3Tests
|
||||
{
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
|
||||
private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text };
|
||||
private static string SchemaId => CultDocumentRegistry.Shared.GetRequired<NetworkSchemaNote>().SchemaId;
|
||||
|
||||
// Another thread becomes the drainer and parks inside a subscriber on the change "slow".
|
||||
private static (Task Writer, ManualResetEventSlim Release) BlockDrainer(CultNetDatabase database, CultCache cache, out IDisposable subscription)
|
||||
{
|
||||
var entered = new ManualResetEventSlim();
|
||||
var release = new ManualResetEventSlim();
|
||||
subscription = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "slow")
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(Wait);
|
||||
}
|
||||
});
|
||||
var writer = Task.Run(() => cache.UpsertAsync(Note("slow"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul3:slow"))).GetAwaiter().GetResult());
|
||||
Assert.That(entered.Wait(Wait), Is.True, "drainer parked");
|
||||
return (writer, release);
|
||||
}
|
||||
|
||||
// P1: a write that returns has been logged and published.
|
||||
[Test]
|
||||
public async Task P1_AWriteThatReturnedIsLoggedAndPublished()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var seen = new ConcurrentQueue<(string? Text, int Thread)>();
|
||||
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue((change.Document?.Text, Environment.CurrentManagedThreadId)));
|
||||
var (writer, release) = BlockDrainer(database, cache, out var block);
|
||||
var fast = new CultRecordKey("soul3:fast");
|
||||
var writerThread = 0;
|
||||
await Task.Run(async () => { writerThread = Environment.CurrentManagedThreadId; await database.PutAsync(fast, Note("fast")); });
|
||||
|
||||
var logged = database.LastWriteSequence(SchemaId, fast);
|
||||
var published = seen.Any(entry => entry.Text == "fast");
|
||||
release.Set();
|
||||
await writer;
|
||||
block.Dispose();
|
||||
var deliveredOn = seen.First(entry => entry.Text == "fast").Thread;
|
||||
TestContext.Progress.WriteLine($"SOUL P1 logged-at-return={logged} published-at-return={published} writer={writerThread} deliveredOn={deliveredOn}");
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(logged, Is.Not.Null, "PutAsync returned before its change was logged");
|
||||
Assert.That(published, Is.True, "PutAsync returned before its change was published");
|
||||
});
|
||||
}
|
||||
|
||||
// P2: a subscriber that waits (bounded) for another thread's write to be observed.
|
||||
[Test]
|
||||
public async Task P2_ASubscriberWaitingForAnotherThreadsWriteIsNotStarved()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var other = new ManualResetEventSlim();
|
||||
var observedInTime = false;
|
||||
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "other")
|
||||
other.Set();
|
||||
if (change.Document?.Text == "trigger")
|
||||
{
|
||||
Task.Run(() => database.PutAsync(new CultRecordKey("soul3:other"), Note("other")));
|
||||
observedInTime = other.Wait(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
});
|
||||
await database.PutAsync(new CultRecordKey("soul3:trigger"), Note("trigger"));
|
||||
Assert.That(observedInTime, Is.True, "another thread's committed write never reached subscribers while this handler ran");
|
||||
}
|
||||
|
||||
// P3: one observer's escaped exception (fail-fast R3 handler) loses one change; it must not stall all later ones.
|
||||
[Test]
|
||||
public async Task P3_AnEscapedObserverExceptionDoesNotStallLaterChanges()
|
||||
{
|
||||
var previous = ObservableSystem.GetUnhandledExceptionHandler();
|
||||
ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception);
|
||||
try
|
||||
{
|
||||
var cache = new CultCache();
|
||||
using var thrower = cache.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom");
|
||||
});
|
||||
var database = new CultNetDatabase(cache);
|
||||
var seen = new ConcurrentQueue<string?>();
|
||||
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue(change.Document?.Text));
|
||||
try { await cache.UpsertAsync(Note("boom"), new CultRecordHandle<NetworkSchemaNote>(new CultRecordKey("soul3:boom"))); }
|
||||
catch (InvalidOperationException) { }
|
||||
|
||||
ObservableSystem.RegisterUnhandledExceptionHandler(previous);
|
||||
var after = new CultRecordKey("soul3:after");
|
||||
await database.PutAsync(after, Note("after"));
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(seen, Does.Contain("after"), "a later commit was never published");
|
||||
Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged");
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
ObservableSystem.RegisterUnhandledExceptionHandler(previous);
|
||||
}
|
||||
}
|
||||
|
||||
// P4: an admission that throws inside Apply after minting a Sequence (a [CultName] getter) must not stall later changes.
|
||||
[Test]
|
||||
public async Task P4_AThrowMidApplyDoesNotStallLaterChanges()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var seen = new ConcurrentQueue<string?>();
|
||||
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change => seen.Enqueue(change.Document?.Text));
|
||||
try
|
||||
{
|
||||
cache.Commit(batch =>
|
||||
{
|
||||
batch.Upsert(new SoulThrowingName { Text = "ok" }, new CultRecordHandle<SoulThrowingName>(new CultRecordKey("soul3:ok")));
|
||||
batch.Upsert(new SoulThrowingName { Text = "boom" }, new CultRecordHandle<SoulThrowingName>(new CultRecordKey("soul3:boomname")));
|
||||
});
|
||||
}
|
||||
catch (InvalidOperationException) { }
|
||||
|
||||
var after = new CultRecordKey("soul3:after-apply");
|
||||
await database.PutAsync(after, Note("after"));
|
||||
TestContext.Progress.WriteLine($"SOUL P4 ok-in-cache={cache.Get(new CultRecordKey("soul3:ok")) != null}");
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(seen, Does.Contain("after"), "a later commit was never published");
|
||||
Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged");
|
||||
});
|
||||
}
|
||||
|
||||
// P5: writers no longer carry their own delivery; the backlog behind one drainer grows without bound.
|
||||
[Test]
|
||||
public async Task P5_OtherWritersDoNotPileAnUnboundedBacklogOnOneDrainer()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache);
|
||||
var delivered = 0;
|
||||
var backlogAtBDone = -1;
|
||||
const int bWrites = 2000;
|
||||
using var watch = database.Watch<NetworkSchemaNote>().Subscribe(change =>
|
||||
{
|
||||
if (change.Document?.Text?.StartsWith("b") == true) Interlocked.Increment(ref delivered);
|
||||
if (change.Document?.Text == "a")
|
||||
{
|
||||
Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < bWrites; i++)
|
||||
await database.PutAsync(new CultRecordKey($"soul3:b:{i}"), Note($"b{i}"));
|
||||
}).Wait(Wait);
|
||||
backlogAtBDone = bWrites - Volatile.Read(ref delivered);
|
||||
}
|
||||
});
|
||||
await database.PutAsync(new CultRecordKey("soul3:a"), Note("a"));
|
||||
TestContext.Progress.WriteLine($"SOUL P5 undelivered after B's {bWrites} writes returned: {backlogAtBDone}");
|
||||
Assert.That(backlogAtBDone, Is.EqualTo(0));
|
||||
}
|
||||
|
||||
// P6: a prediction still queued when the authoritative put arrives.
|
||||
[Test]
|
||||
public async Task P6_AnAuthoritativePutArrivingBeforeItsPredictionIsDeliveredStillReconciles()
|
||||
{
|
||||
var cache = new CultCache();
|
||||
var database = new CultNetDatabase(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
RuntimeId = "local",
|
||||
ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")]
|
||||
});
|
||||
var key = new CultRecordKey("soul3:predict");
|
||||
var kinds = new ConcurrentQueue<CultNetDatabaseChangeKind>();
|
||||
using var watch = database.WatchRecord<NetworkSchemaNote>(key).Subscribe(change => kinds.Enqueue(change.Kind));
|
||||
CultNetDocumentPutRawMessage Put(string text)
|
||||
{
|
||||
var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle<NetworkSchemaNote>(key), Note(text));
|
||||
message.ShardId = database.Shards[0].ShardId;
|
||||
message.ShardEpoch = database.Shards[0].Epoch;
|
||||
return message;
|
||||
}
|
||||
|
||||
var (writer, release) = BlockDrainer(database, cache, out var block);
|
||||
await database.PutPredictedAsync(key, Note("predicted"));
|
||||
await database.ApplyPutAsync(Put("auth1"));
|
||||
release.Set();
|
||||
await writer;
|
||||
block.Dispose();
|
||||
await database.ApplyPutAsync(Put("auth2"));
|
||||
|
||||
Assert.That(kinds, Is.EqualTo(new[]
|
||||
{
|
||||
CultNetDatabaseChangeKind.Predicted,
|
||||
CultNetDatabaseChangeKind.Reconciled,
|
||||
CultNetDatabaseChangeKind.Updated
|
||||
}));
|
||||
}
|
||||
|
||||
// P7: a replica's absent-key delete is logged ahead of an earlier entry still queued; a bounded pull then sees a gap.
|
||||
[Test]
|
||||
public async Task P7_AReplicaLogStaysInSequenceOrderWhenAnAbsentDeleteOvertakesAQueuedPut()
|
||||
{
|
||||
const string shardId = "chain3";
|
||||
CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions
|
||||
{
|
||||
Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [SchemaId])]
|
||||
});
|
||||
var p = Make(new CultCache(), true);
|
||||
var rCache = new CultCache();
|
||||
var r = Make(rCache, false);
|
||||
var c = Make(new CultCache(), false);
|
||||
async Task Pull(CultNetDatabase from, CultNetDatabase to, int? limit = null)
|
||||
{
|
||||
await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage
|
||||
{
|
||||
ShardId = shardId,
|
||||
ShardEpoch = 2,
|
||||
Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId), limit).ToArray()
|
||||
});
|
||||
}
|
||||
|
||||
var z = new CultRecordKey("soul3:z");
|
||||
await p.PutAsync(z, Note("z"));
|
||||
await Pull(p, r);
|
||||
await Pull(r, c);
|
||||
rCache.Remove(z);
|
||||
await p.PutAsync(new CultRecordKey("soul3:a"), Note("a"));
|
||||
await p.DeleteAsync<NetworkSchemaNote>(z);
|
||||
|
||||
var (writer, release) = BlockDrainer(r, rCache, out var block);
|
||||
await Pull(p, r);
|
||||
release.Set();
|
||||
await writer;
|
||||
block.Dispose();
|
||||
|
||||
TestContext.Progress.WriteLine($"SOUL P7 r log order: {string.Join(",", r.GetMutationLog(shardId).Select(e => e.Sequence))}");
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(r.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }));
|
||||
Assert.DoesNotThrowAsync(() => Pull(r, c, limit: 1), "a bounded pull from R");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue