diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 8d2172c0..66bb450e 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -104,6 +104,16 @@ Not implemented yet: - cross-runtime rollback/resimulation helpers for simulation frames - declared CRDT merge policies +## Error Codes + +`cultnet.error.v0` carries an optional machine-readable `code`. The set of codes is a closed enum +(`cultnet.error.schema.json`): Rust and TypeScript refuse a message whose code they do not know, they do +not guess. Adding a code is therefore a coordinated change across every runtime, made together: the schema (both +copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in +`packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are +`selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported` and +`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns). + ## Live Invariants - CultCache owns document schema, identity, local indexes, and local diffing. diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index 15880fa1..59764338 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -26,9 +26,9 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index 15880fa1..59764338 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -26,9 +26,9 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index f31ade85..1808c62f 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -250,8 +250,8 @@ impl CultNetShardDescriptor { } } -/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings -/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth +/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the strings +/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a /// string it does not know refuses to decode rather than guess a meaning for it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -261,6 +261,7 @@ pub enum CultNetErrorCode { CursorInvalid, ReferenceOutsideTarget, VariantUnsupported, + UnownedSchema, } impl CultNetErrorCode { @@ -271,6 +272,7 @@ impl CultNetErrorCode { Self::CursorInvalid => "cursor_invalid", Self::ReferenceOutsideTarget => "reference_outside_target", Self::VariantUnsupported => "variant_unsupported", + Self::UnownedSchema => "unowned_schema", } } @@ -281,6 +283,7 @@ impl CultNetErrorCode { "cursor_invalid" => Ok(Self::CursorInvalid), "reference_outside_target" => Ok(Self::ReferenceOutsideTarget), "variant_unsupported" => Ok(Self::VariantUnsupported), + "unowned_schema" => Ok(Self::UnownedSchema), other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")), } } diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 011ff147..5c633087 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -118,6 +118,46 @@ fn cases() -> Vec { details: None, }, }, + Case { + name: "variant_unsupported_subscription_ended", + // C#: ForVariantUnsupported("variant unsupported here", "sub-1") + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: Some(CultNetErrorDetails { + field: Some("subscriptionId".to_string()), + value: Some("sub-1".to_string()), + as_of: None, + current: None, + }), + }, + }, + Case { + name: "variant_unsupported_no_subscription", + // C#: ForVariantUnsupported("variant unsupported here") - no subscription ended, so Details stays null. + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: None, + }, + }, + Case { + name: "unowned_schema", + // C#: ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))) + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.".to_string(), + code: Some(CultNetErrorCode::UnownedSchema), + details: Some(CultNetErrorDetails { + field: Some("schemaId".to_string()), + value: Some("some.schema.v1".to_string()), + as_of: None, + current: None, + }), + }, + }, ] } diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index a70814d5..a19a8839 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -47,10 +47,20 @@ which route wrote it. - Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a per-cache, in-memory number assigned when the cache admits the change, under its gate. It increases in admission order and is not persisted. - `OnUpdate(previous, current)` carries none. Streams CultNet derives from - `OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server, - database-backed Mesh handles) have no ordering guarantee and no stale - protection. + `OnUpdate(previous, current)` carries none. CultNet derives its streams + (`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed + Mesh handles) from `Watch` plus one journal (`AddJournal`), so a load, a + commit and a variant dependent's re-resolution all reach subscribers and the + mutation log. The journal runs under the gate in `Sequence` order, so the + shard log is written in cache order; the `Watch` observer publishes what the + journal decided. A database write door only hands the journal the wire message + or replicated entry for the change it admitted. A replica publishes but never + mints log sequences. A primary logs every change it admits, loads and pulls + included; a store attached after the database exists has its hydration logged + too. A primary that cannot log a change compacts through that change's + sequence and throws `CultNetShardLogException` to the writer after publication: + the commit stands, and a replica behind it resynchronizes from a snapshot. The + streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at or below it is reflected in the document. @@ -126,6 +136,10 @@ which route wrote it. needs the cache or its gate: another thread's write, a lock a writer holds. Its own leaf locks, taken only inside the journal, are the only locks that may nest under the gate. +- A shard mutation log store (`ICultNetShardMutationLogStore`) runs inside the + CultNet journal, under the cache's gate: it must not call the cache or wait + for a thread that does. A store that writes its own cache throws through the + journal's reentry guard, and the admission stands. - A thread holds one cache's gate at a time: entering a cache's hold inside another cache's hold throws. - A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index e8ec8fd6..6365261f 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2124,7 +2124,15 @@ namespace GameCult.Caching Remove(handle.Key); } - // false: a condition failed; nothing was written, changed in memory, or published. + /// + /// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published. + /// + /// + /// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer + /// receives the failure. One failure arrives as itself, several as an . A writer that + /// wants a particular failure (a CultNetShardLogException, when a database's log refused the commit) catches that type, + /// or unwraps and looks for it. + /// public bool Commit(Action stage) { return Land(stage, wait: true) == CultCommitOutcome.Committed; @@ -2427,7 +2435,8 @@ namespace GameCult.Caching // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not // guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate, // in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even - // if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown. + // if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their + // origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions. // If the body threw, its exception wins. internal T Held(Func body) { diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 52c67482..ef41d1f7 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -3379,7 +3379,8 @@ namespace GameCult.Mesh if (document != null) return document; - var untyped = database.Cache.Get(key); + // Through the database, which serves only what its shards own; the cache behind it may hold more. + var untyped = await database.GetAsync(key).ConfigureAwait(false); if (untyped != null && IsSameCultDocumentSchema(untyped.GetType())) return ConvertUntypedDocument(untyped); diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index d59cd6c7..668c088e 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using GameCult.Caching; @@ -435,6 +437,10 @@ namespace GameCult.Networking /// Gets the previous document, when present. /// public object? PreviousDocument { get; } + + // The primary's own wire entry, kept for an entry this replica changed nothing for (a row its shard does not own, an + // absent key): it is relayed downstream exactly as the primary sent it, in memory and on disk alike. + internal CultNetShardLogEntryMessage? Relayed { get; set; } } /// @@ -461,6 +467,101 @@ namespace GameCult.Networking public string Reason { get; } } + /// + /// One shard-log sequence that a primary minted for a committed change and burned because it could not log it. + /// + public readonly struct CultNetBurnedSequence : IEquatable + { + /// + /// Creates a burned sequence. + /// + public CultNetBurnedSequence(string shardId, long sequence) + { + ShardId = shardId; + Sequence = sequence; + } + + /// + /// Gets the shard whose log burned the sequence. + /// + public string ShardId { get; } + + /// + /// Gets the burned shard-log sequence. + /// + public long Sequence { get; } + + /// + public bool Equals(CultNetBurnedSequence other) => Sequence == other.Sequence && string.Equals(ShardId, other.ShardId, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is CultNetBurnedSequence other && Equals(other); + + /// + public override int GetHashCode() => (StringComparer.Ordinal.GetHashCode(ShardId ?? string.Empty) * 397) ^ Sequence.GetHashCode(); + + /// + public override string ToString() => $"{ShardId}:{Sequence}"; + } + + /// + /// Raised when a database write, or a resolution, names a schema and key that no configured shard owns. A refused write + /// caches nothing. + /// + public sealed class CultNetUnownedSchemaException : InvalidOperationException + { + /// + /// Creates an unowned schema exception. + /// + public CultNetUnownedSchemaException(string schemaId, CultRecordKey key) + : base($"No shard owns schema '{schemaId}' key '{key.Value}'; the write was refused.") + { + SchemaId = schemaId; + Key = key; + } + + /// + /// Gets the schema that no shard owns. + /// + public string SchemaId { get; } + + /// + /// Gets the key of the refused write. + /// + public CultRecordKey Key { get; } + } + + /// + /// Raised to the writer when a shard primary committed changes and could not log them. The commits stand and the + /// changes were published; each shard's log burned the sequences in and compacted past them, + /// so every replica behind one resynchronizes from a snapshot. A single journal call raises one of these carrying + /// every burned sequence, whatever number of changes or shards failed. It is deliberately not an + /// : callers that read that type as a refused write must not mistake a + /// committed one for it. + /// + /// + /// The cache may still wrap this in an when something else failed in the same hold + /// (an observer, a second database's journal). A writer that wants the log failure catches this type, or unwraps + /// and looks for it. is the + /// store's failure, or an of them when several changes failed. + /// + public sealed class CultNetShardLogException : Exception + { + /// + /// Creates a shard log exception. + /// + public CultNetShardLogException(IReadOnlyList burned, Exception cause) + : base($"Committed changes could not be logged; sequences {string.Join(", ", burned.Select(entry => $"{entry.ShardId}:{entry.Sequence}"))} are burned and compacted past.", cause) + { + Burned = burned; + } + + /// + /// Gets every sequence burned by the journal call, in commit order. + /// + public IReadOnlyList Burned { get; } + } + /// /// Database-style CultNet facade over a CultCache shard set. /// @@ -472,18 +573,39 @@ namespace GameCult.Networking private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; - private readonly Dictionary _predictedDocuments = new(StringComparer.Ordinal); - private readonly Dictionary> _mutationLogs = + // Keyed by sequence, like the durable store: recording an entry twice replaces it. + private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); + // The highest sequence this primary burned per shard (a change it committed and could not log). Serves as the + // shard's compaction floor beside the store's own, so history the log cannot serve is answered compacted_history. + private readonly Dictionary _burnedThrough = new(StringComparer.Ordinal); private readonly Dictionary _appliedShardSequences = new(StringComparer.Ordinal); // CultNet typed selection, section 2: "ordinal is the sequence of the commit that last wrote - // the row". Kept on every append (AppendMutationLogEntry is the one site), keyed by (schemaId, + // the row". Kept on every append (LogPrimaryChange and RecordReplicatedEntry are the sites), keyed by (schemaId, // recordKey) so the evaluator's order and cursor never read a clock. private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); - private bool _disposed; + private readonly IDisposable _cacheChanges; + private readonly IDisposable _cacheJournal; + // What a write door of this database knows about the change it is admitting and the cache does not: the wire + // message it carries, a prediction it reconciles, the replicated log entry it applies. The journal is the only + // logger and consumes the context under the cache gate; a door registers it under the identity of the change it + // admits (the instance it writes, or the instance it removes), and the journal takes it once, for that change + // alone. A write to the same key by anyone else is a different instance and gets no context. + private readonly ConcurrentDictionary _doors = new(ReferenceIdentity.Instance); + // One gate over the mutation log, its sequence counters, the last-write map and the publication stash. Lock order: + // the cache gate, then this, then the log store's own lock. Nothing holding this calls the cache, and a log store + // must not either (ICultNetShardMutationLogStore). Publication stays outside it: a subscriber may write the cache + // from its handler. + private readonly object _logGate = new(); + // Changes the journal has decided and the Watch observer has not yet published, by cache Sequence. The journal + // runs under the cache gate before any observer, so each change is stashed before it can be published. + private readonly Dictionary _stash = new(); + // Predictions this runtime made and has not seen reconciled. Only the journal reads or writes it, under the cache gate. + private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); + private volatile bool _disposed; /// /// Creates a database surface over a CultCache instance. @@ -502,7 +624,9 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - _cache.OnUpdate += PublishCacheUpdate; + // The observer first, the journal second: a change the journal stashed always finds its publisher. + _cacheChanges = _cache.Watch().Subscribe(Publish); + _cacheJournal = _cache.AddJournal(Journal); } /// @@ -536,7 +660,7 @@ namespace GameCult.Networking { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - return ResolveShardInternal(descriptor, key).IsPrimary; + return FindShard(descriptor, key)?.IsPrimary ?? false; } /// Gets whether this database instance can submit a prediction for the document at a key. @@ -557,18 +681,21 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - if (!_mutationLogs.TryGetValue(shardId, out var entries)) + lock (_logGate) { - return Array.Empty(); - } + if (!_mutationLogs.TryGetValue(shardId, out var entries)) + { + return Array.Empty(); + } - var query = entries.Where(entry => entry.Sequence > afterSequence); - if (limit.HasValue) - { - query = query.Take(limit.Value); - } + var query = entries.Values.Where(entry => entry.Sequence > afterSequence); + if (limit.HasValue) + { + query = query.Take(limit.Value); + } - return query.ToArray(); + return query.ToArray(); + } } /// @@ -598,7 +725,15 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _mutationLogStore?.GetCompactedThrough(shardId) ?? 0; + lock (_logGate) + return CompactedFloor(shardId); + } + + // The store's compaction point or this primary's own burned sequences, whichever is higher. Under _logGate. + private long CompactedFloor(string shardId) + { + var burned = _burnedThrough.TryGetValue(shardId, out var through) ? through : 0; + return Math.Max(burned, _mutationLogStore?.GetCompactedThrough(shardId) ?? 0); } /// @@ -608,11 +743,13 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - var compactedThrough = GetCompactedMutationLogSequence(shardId); - var retained = _mutationLogStore != null - ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) - : GetMutationLog(shardId).Select(entry => entry.Sequence); - return retained.DefaultIfEmpty(compactedThrough).Max(); + lock (_logGate) + { + var retained = _mutationLogStore != null + ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) + : GetMutationLog(shardId).Select(entry => entry.Sequence); + return retained.Append(CompactedFloor(shardId)).Max(); + } } /// @@ -629,6 +766,10 @@ namespace GameCult.Networking ThrowIfDisposed(); if (shard == null) throw new ArgumentNullException(nameof(shard)); + // The log sequence is read before any document: a write that lands in between is then replayed from the log on + // top of documents that already hold it (whole-document puts and deletes, applied in order), and a snapshot can + // never claim a sequence whose write its documents lack. + var logSequence = GetLatestMutationLogSequence(shard.ShardId); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); var lowKeys = CultNetV0SelectionLowering.Lower(filter?.RecordKeys); @@ -642,7 +783,7 @@ namespace GameCult.Networking Documents = Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -665,7 +806,7 @@ namespace GameCult.Networking Documents = page.Documents ?? Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -700,39 +841,17 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - RemoveTrackedDocument(existing.DocumentType, existing.Key); - PublishUntyped( - existing.DocumentType, - CultNetDatabaseChangeKind.Removed, - existing.Key, - existing.SchemaId, - shard, - document: null, - previousDocument: existing.Document); + using (Door(existing.Document, new DoorContext(shard) { NoLog = true, Removal = true })) + _cache.Remove(existing.Key); } foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document))) { var key = new CultRecordKey(document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(document); - var previous = _cache.Get(key); - var applied = await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document, - ShardId = shard.ShardId, - ShardEpoch = shard.Epoch - }).ConfigureAwait(false); - PublishUntyped( - descriptor.DocumentType, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - applied, - previous); + var applied = _documents.DeserializeRawDocument(document); + using (Door(applied, new DoorContext(shard) { NoLog = true })) + await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false); } var sequence = response.ShardLogSequence ?? 0; @@ -792,6 +911,7 @@ namespace GameCult.Networking /// /// Resolves the shard that governs the supplied schema and key. /// + /// No configured shard owns the schema and key. public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key) { ThrowIfDisposed(); @@ -804,16 +924,18 @@ namespace GameCult.Networking public Task GetAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); - return Task.FromResult(_cache.Get(key)); + var document = _cache.Get(key); + return Task.FromResult(document != null && Owns(_cache.Registry.GetRequired(document.GetType()), key) ? document : null); } /// - /// Gets all documents assignable to the requested type. + /// Gets all documents assignable to the requested type. A database serves only what its shards own: a cached row no + /// shard owns is left out here, as it is from every read and projection. The cache itself still holds it. /// public IEnumerable GetAll() where T : class { ThrowIfDisposed(); - return _cache.GetAll(); + return _cache.GetAll().Where(Serves); } /// @@ -822,7 +944,8 @@ namespace GameCult.Networking public T? GetByName(string name) where T : class { ThrowIfDisposed(); - return _cache.GetByName(name); + var document = _cache.GetByName(name); + return document != null && Serves(document) ? document : null; } /// @@ -831,47 +954,30 @@ namespace GameCult.Networking public T? GetByIndex(string alias, string value) where T : class { ThrowIfDisposed(); - return _cache.GetByIndex(alias, value); + var document = _cache.GetByIndex(alias, value); + return document != null && Serves(document) ? document : null; } /// /// Adds or replaces a document at a specific key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task> PutAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(document.GetType()); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - void PublishCommittedPut() - { - AppendMutationLogEntry( - shard, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - document, - previous); - Publish(new CultNetDatabaseChange( - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - document, - previous)); - } - PublishCommittedPut(); - return handle; + return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } /// /// Applies a locally predicted document for a client-owned input scope. /// + /// No shard owns the schema and key; nothing was written. public async Task> PutPredictedAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); @@ -881,59 +987,38 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsureClientAuthority(descriptor, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Predicted, - key, - descriptor.SchemaId, - shard, - document, - previous)); + CultRecordHandle handle; + using (Door(document, new DoorContext(shard) + { + Kind = CultNetDatabaseChangeKind.Predicted, + NoLog = true, + PredictionKey = PredictionKey(descriptor.SchemaId, key), + Predicts = true + })) + handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); return handle; } /// /// Deletes a document by key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task DeleteAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); - - var previous = _cache.Get(key); - if (previous != null) - { - _cache.Remove(new CultRecordHandle(key)); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous); - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous)); - } - PublishCommittedDelete(); - } + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); + _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; } /// /// Applies a raw document mutation after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task ApplyPutAsync(CultNetDocumentPutRawMessage message) { ThrowIfDisposed(); @@ -947,48 +1032,17 @@ namespace GameCult.Networking var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); - var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; - var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); - if (reconcilesPrediction) - { - kind = CultNetDatabaseChangeKind.Reconciled; - } - - void PublishCommittedPut() - { - if (reconcilesPrediction) - _predictedDocuments.Remove(predictionKey); - AppendMutationLogEntry( - shard, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - key, - descriptor.SchemaId, - document, - previous, - new CultNetShardLogEntryMessage - { - ChangeKind = previous == null ? "added" : "updated", - Put = message - }); - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); - } - PublishCommittedPut(); + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Put = message, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); return document; } /// /// Applies a raw document delete after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task ApplyDeleteAsync(CultNetDocumentDeleteMessage message) { ThrowIfDisposed(); @@ -1004,31 +1058,8 @@ namespace GameCult.Networking return Task.CompletedTask; } - _cache.Remove(key); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous, - wireEntry: new CultNetShardLogEntryMessage - { - ChangeKind = "removed", - Delete = message - }); - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); - } - PublishCommittedDelete(); + using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true })) + _cache.Remove(key); return Task.CompletedTask; } @@ -1142,7 +1173,6 @@ namespace GameCult.Networking (change.Document != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.Document)) || (change.PreviousDocument != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.PreviousDocument))); } - /// /// Releases database subscriptions. /// @@ -1154,40 +1184,318 @@ namespace GameCult.Networking } _disposed = true; - _cache.OnUpdate -= PublishCacheUpdate; + // The journal first: disposing it waits for a journal in flight, so nothing is stashed after this line. What a + // writer has logged and not yet published is then published here, so a change the log holds is never dropped. + _cacheJournal.Dispose(); + _cacheChanges.Dispose(); + Publication[] unpublished; + lock (_logGate) + { + unpublished = _stash.OrderBy(entry => entry.Key).Select(entry => entry.Value).ToArray(); + _stash.Clear(); + } + + foreach (var publication in unpublished) + { + Publish(publication); + } + _changes.Dispose(); } - private void PublishCacheUpdate(object? previous, object? current) + // The cache is the one owner of "a change was committed"; the journal is this database's one decision point and its + // one logger. It runs under the cache gate, in cache Sequence order, before any observer: it takes the change's door, + // settles the change kind and the prediction bookkeeping, logs the change when this runtime is the shard's primary + // (a replica records only the primary's own entry, which its door carries), and stashes what the publisher needs. + // A change with no door was written by someone else (a bare cache write, a load, a pull) and is logged like any + // unattributed commit. It never enters the cache and calls nothing that does. + private void Journal(IReadOnlyList> changes) { - var document = current ?? previous; - if (document == null) + if (_disposed) { return; } - var documentType = document.GetType(); - var descriptor = _cache.Registry.GetRequired(documentType); - var key = current != null - ? GetTrackedKey(current, documentType) - : new CultRecordKey(string.Empty); - var shard = ResolveShardInternal(descriptor, key); - var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); - var change = Activator.CreateInstance( - changeType, - current == null ? CultNetDatabaseChangeKind.Removed : - previous == null ? CultNetDatabaseChangeKind.Added : - CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - current, - previous, - null); - if (change != null) + List? failures = null; + List? burned = null; + List? burnCauses = null; + foreach (var change in changes) { - _changes.OnNext(change); + var identity = change.Document ?? change.PreviousDocument; + if (identity == null) + { + continue; + } + + var door = TakeDoor(change); + var documentType = identity.GetType(); + var descriptor = _cache.Registry.GetRequired(documentType); + var shard = door?.Shard ?? FindShard(descriptor, change.Key); + if (shard == null) + { + // No shard owns a bare cache write of this schema: the database neither logs nor publishes it. + continue; + } + + var kind = door?.Kind ?? change.Kind switch + { + CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, + CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, + _ => CultNetDatabaseChangeKind.Updated + }; + if (door?.PredictionKey is { } predictionKey) + { + if (door.Predicts) + _predictedDocuments.Add(predictionKey); + else if (_predictedDocuments.Remove(predictionKey)) + kind = CultNetDatabaseChangeKind.Reconciled; + } + + // The log records a reconciled change as the update it is. + var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + try + { + if (door?.Replica is { } replicated) + { + RecordReplicatedEntry(shard, logKind, descriptor.SchemaId, change, replicated); + } + else if (door?.NoLog != true && shard.IsPrimary) + { + if (LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door) is { } burn) + { + (burned ??= new List()).Add(new CultNetBurnedSequence(shard.ShardId, burn.Sequence)); + (burnCauses ??= new List()).Add(burn.Cause); + } + } + } + catch (Exception exception) + { + (failures ??= new List()).Add(exception); + } + + lock (_logGate) + _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } + + // Every change is stashed and will be published; the writer hears of a failed log afterwards, in one exception. + if (burned != null) + { + (failures ??= new List()).Insert( + 0, + new CultNetShardLogException(burned, burnCauses!.Count == 1 ? burnCauses[0] : new AggregateException(burnCauses))); + } + + if (failures != null) + { + throw failures.Count == 1 ? failures[0] : new AggregateException(failures); + } + } + + // The Watch observer publishes, on the writer's thread after the cache gate opens and before the write returns, what + // the journal stashed for this change. A change with no stash was admitted before this database existed. + private void Publish(CultCacheDocumentChange change) + { + Publication? publication; + lock (_logGate) + { + if (!_stash.Remove(change.Sequence, out publication)) + { + return; + } + } + + Publish(publication); + } + + private void Publish(Publication publication) => + PublishUntyped( + publication.DocumentType, + publication.Kind, + publication.Key, + publication.SchemaId, + publication.Shard, + publication.Document, + publication.PreviousDocument); + + private void RecordReplicatedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + CultNetShardLogEntryMessage replicated) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + replicated.Sequence, + replicated.CommittedAt, + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument), + replicated); + _lastWriteSequence[(schemaId, change.Key.Value)] = replicated.Sequence; + } + } + + // A primary mints the sequence, appends durably, then records in memory. If it cannot log the change, the change is + // still committed: the sequence is burned and compacted past, so a replica behind it is told compacted_history and + // resynchronizes by snapshot. The burn is returned for the journal to tell the writer in one CultNetShardLogException. + private (long Sequence, Exception Cause)? LogPrimaryChange( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + DoorContext? door) + { + lock (_logGate) + { + var sequence = NextMutationLogSequence(shard.ShardId); + _lastWriteSequence[(schemaId, change.Key.Value)] = sequence; + try + { + var entry = new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + sequence, + DateTimeOffset.UtcNow.ToString("O"), + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument); + // A door's own wire message is what the log stores (its kind, sequence and shard are restated from the entry). + var wireEntry = _mutationLogStore == null + ? null + : door is { Put: not null } or { Delete: not null } + ? new CultNetShardLogEntryMessage { Put = door.Put, Delete = door.Delete } + : ToLogEntryMessage(entry); + RecordMutationLogEntry(entry, wireEntry); + return null; + } + catch (Exception cause) + { + _burnedThrough[shard.ShardId] = Math.Max(_burnedThrough.TryGetValue(shard.ShardId, out var burned) ? burned : 0, sequence); + try + { + _mutationLogStore?.CompactThrough(shard.ShardId, sequence); + } + catch (Exception compactCause) + { + cause = new AggregateException(cause, compactCause); + } + + return (sequence, cause); + } + } + } + + // A door's context belongs to one specific change, recognised by shape: a put door's change carries the instance as + // its new document; a removal door removes an instance, so its change carries it as the previous document and no + // new one. A plain removal of an instance a put door admitted is neither, and gets no context. A door's own change + // is journaled first, in its own hold, and takes its context there. + private DoorContext? TakeDoor(CultCacheDocumentChange change) + { + var identity = change.Document ?? change.PreviousDocument; + if (identity == null || !_doors.TryGetValue(identity, out var door)) + { + return null; + } + + var matches = change.Document == null + ? door.Removal + : !door.Removal; + return matches && ((ICollection>)_doors) + .Remove(new KeyValuePair(identity, door)) + ? door + : null; + } + + private DoorRegistration Door(object identity, DoorContext context) => + new(_doors, identity, context, _doors.TryAdd(identity, context)); + + // Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind. + private readonly struct DoorRegistration : IDisposable + { + private readonly ConcurrentDictionary _doors; + private readonly object _identity; + private readonly DoorContext _context; + private readonly bool _owned; + + public DoorRegistration(ConcurrentDictionary doors, object identity, DoorContext context, bool owned) + { + _doors = doors; + _identity = identity; + _context = context; + _owned = owned; + } + + // True when the journal took the context: the door's write was admitted. + public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context); + + public void Dispose() + { + if (_owned) + ((ICollection>)_doors).Remove(new KeyValuePair(_identity, _context)); + } + } + + // What the publisher needs of a change the journal decided. + private sealed class Publication + { + public Publication( + Type documentType, + CultNetDatabaseChangeKind kind, + CultRecordKey key, + string schemaId, + CultNetShardDescriptor shard, + object? document, + object? previousDocument) + { + DocumentType = documentType; + Kind = kind; + Key = key; + SchemaId = schemaId; + Shard = shard; + Document = document; + PreviousDocument = previousDocument; + } + + public Type DocumentType { get; } + public CultNetDatabaseChangeKind Kind { get; } + public CultRecordKey Key { get; } + public string SchemaId { get; } + public CultNetShardDescriptor Shard { get; } + public object? Document { get; } + public object? PreviousDocument { get; } + } + + private sealed class DoorContext + { + public DoorContext(CultNetShardDescriptor shard) => Shard = shard; + + public CultNetShardDescriptor Shard { get; } + public CultNetDatabaseChangeKind? Kind { get; set; } + public bool NoLog { get; set; } + public bool Removal { get; set; } + public CultNetDocumentPutRawMessage? Put { get; set; } + public CultNetDocumentDeleteMessage? Delete { get; set; } + public CultNetShardLogEntryMessage? Replica { get; set; } + // The prediction this door's change either records (Predicts) or ends. + public string? PredictionKey { get; set; } + public bool Predicts { get; set; } + } + + private sealed class ReferenceIdentity : IEqualityComparer + { + public static readonly ReferenceIdentity Instance = new(); + + bool IEqualityComparer.Equals(object? x, object? y) => ReferenceEquals(x, y); + + int IEqualityComparer.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value); } private CultRecordKey GetTrackedKey(object document, Type documentType) @@ -1240,16 +1548,6 @@ namespace GameCult.Networking } } - private void RemoveTrackedDocument(Type documentType, CultRecordKey key) - { - _cache.Remove(key); - } - - private void Publish(CultNetDatabaseChange change) where T : class - { - _changes.OnNext(change); - } - private sealed class TrackedShardDocument { public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document) @@ -1266,41 +1564,17 @@ namespace GameCult.Networking public object Document { get; } } - private void AppendMutationLogEntry( - CultNetShardDescriptor shard, - CultNetDatabaseChangeKind kind, - CultRecordKey key, - string schemaId, - object? document, - object? previousDocument, - CultNetShardLogEntryMessage? wireEntry = null) - { - var sequence = NextMutationLogSequence(shard.ShardId); - var committedAt = DateTimeOffset.UtcNow.ToString("O"); - var entry = new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - sequence, - committedAt, - kind, - schemaId, - key, - document, - previousDocument); - var storedWireEntry = _mutationLogStore == null - ? null - : wireEntry ?? ToLogEntryMessage(entry); - RecordMutationLogEntry(entry, storedWireEntry); - _lastWriteSequence[(schemaId, key.Value)] = sequence; - } /// /// The shard-log sequence of the commit that last wrote this row, or null when the row has /// never been committed through this database. The order the CultNet selection evaluator /// pages by (docs/cultnet-selection-cut.md, section 2). /// - public long? LastWriteSequence(string schemaId, CultRecordKey key) => - _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + public long? LastWriteSequence(string schemaId, CultRecordKey key) + { + lock (_logGate) + return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + } /// /// The current write-sequence watermark across every shard this database holds, 0 when nothing @@ -1311,7 +1585,11 @@ namespace GameCult.Networking /// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to /// while the page's own shard sat still. /// - public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + public ulong CurrentAsOf() + { + lock (_logGate) + return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + } /// /// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit @@ -1325,7 +1603,8 @@ namespace GameCult.Networking public ulong CurrentAsOf(string shardId) { if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; + lock (_logGate) + return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; } private void RecordMutationLogEntry( @@ -1334,21 +1613,22 @@ namespace GameCult.Networking { if (!_mutationLogs.TryGetValue(entry.ShardId, out var entries)) { - entries = new List(); + entries = new SortedList(); _mutationLogs[entry.ShardId] = entries; } - entries.Add(entry); + // Disk first: memory never serves an entry the durable log lacks, and a refused append records nothing. + if (_mutationLogStore != null && wireEntry != null) + { + _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); + } + + entries[entry.Sequence] = entry; if (!_nextLogSequences.TryGetValue(entry.ShardId, out var next) || next <= entry.Sequence) { _nextLogSequences[entry.ShardId] = entry.Sequence + 1; } - - if (_mutationLogStore != null && wireEntry != null) - { - _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); - } } private long NextMutationLogSequence(string shardId) @@ -1371,12 +1651,23 @@ namespace GameCult.Networking { // R-Q: the log is the durable record of every commit, replica-applied ones included, so // the last-write sequence a restart resumes with is rebuilt from it exactly as - // AppendMutationLogEntry/the replica apply paths maintain it live - a row's ordinal for + // LogPrimaryChange/the replica apply paths maintain it live - a row's ordinal for // the selection evaluator's order and cursor must not reset to "never written" on restart. - var entries = _mutationLogStore.Read(shard.ShardId); + var entries = _mutationLogStore.Read(shard.ShardId).OrderBy(entry => entry.Sequence).ToArray(); + var compacted = _mutationLogStore.GetCompactedThrough(shard.ShardId); var highest = 0L; + var expected = compacted + 1; + var hole = 0L; foreach (var entry in entries) { + // A sequence the log lacks between two it holds was burned (its append and its compaction both failed, or + // a compaction was interrupted). Replicas must be told compacted_history for it, not served a gapped log. + if (entry.Sequence > compacted && entry.Sequence > expected) + { + hole = entry.Sequence - 1; + } + + expected = Math.Max(expected, entry.Sequence + 1); if (entry.Sequence > highest) { highest = entry.Sequence; @@ -1389,6 +1680,13 @@ namespace GameCult.Networking } } + // Sequences at or below the compaction point are spent even when the log holds none of them. + highest = Math.Max(highest, compacted); + if (hole > 0) + { + _burnedThrough[shard.ShardId] = hole; + } + if (highest > 0) { _nextLogSequences[shard.ShardId] = highest + 1; @@ -1453,8 +1751,6 @@ namespace GameCult.Networking var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { @@ -1462,36 +1758,18 @@ namespace GameCult.Networking $"Shard '{shard.ShardId}' log entry {entry.Sequence} has a removed kind with a put payload."); } - var predictionKey = PredictionKey(descriptor.SchemaId, key); - if (_predictedDocuments.Remove(predictionKey)) + // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that + // admits the change; the journal does both. A retry after a refused append is a fresh instance, so it matches + // its door again and re-records the entry. + if (!shard.Matches(descriptor, key)) { - kind = CultNetDatabaseChangeKind.Reconciled; + RecordUnappliedEntry(shard, kind, descriptor.SchemaId, key, entry); + return; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - descriptor.SchemaId, - key, - document, - previous), - entry); - // R-Q: a replica apply is a commit path too - the last-write sequence this row now carries - // is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets - // it, so the selection evaluator's order and cursor agree whether a row's most recent write - // landed locally or arrived through replication. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } private void ApplyCommittedDelete( @@ -1510,37 +1788,53 @@ namespace GameCult.Networking var key = new CultRecordKey(message.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); + if (!shard.Matches(descriptor, key)) + { + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + return; + } + + var admitted = false; var previous = _cache.Get(key); if (previous != null) { + using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true }); _cache.Remove(key); + admitted = door.Consumed; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - CultNetDatabaseChangeKind.Removed, - descriptor.SchemaId, - key, - document: null, - previousDocument: previous), - entry); - // R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); + if (!admitted) + { + // The key is not here, so nothing was admitted to log. + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + } + } + + // An entry that changed nothing here (an absent key, or a row this shard does not own - the snapshot path leaves such + // rows out too). This replica's log must still hold every entry of the primary's, or a replica chained on it sees a gap. + private void RecordUnappliedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind kind, + string schemaId, + CultRecordKey key, + CultNetShardLogEntryMessage entry) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, kind, + schemaId, key, document: null, previousDocument: null) { Relayed = entry }, entry); + _lastWriteSequence[(schemaId, key.Value)] = entry.Sequence; + } } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) { + if (entry.Relayed != null) + { + return NormalizeWireLogEntry(entry, entry.Relayed); + } + if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null) { return new CultNetShardLogEntryMessage @@ -1637,20 +1931,6 @@ namespace GameCult.Networking }; } - private void EnsureClientAuthority(string schemaId, CultRecordKey key) - { - if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, schemaId, key))) - { - return; - } - - var shard = ResolveShardInternal(schemaId, key); - throw new CultNetShardAuthorityException( - shard, - $"Runtime '{_runtimeId}' does not have client prediction authority for schema '{schemaId}' key '{key.Value}'.", - "not_client_authority"); - } - private void EnsureClientAuthority(CultDocumentDescriptor descriptor, CultRecordKey key) { if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, descriptor, key))) @@ -1695,15 +1975,30 @@ namespace GameCult.Networking } } - private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) + // No shard owns a schema and key that no descriptor matches, and nothing is written to one: every write path refuses. + private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? throw new CultNetUnownedSchemaException(schemaId, key); + + private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) => + FindShard(descriptor, key) ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); + + internal bool Owns(CultDocumentDescriptor descriptor, CultRecordKey key) => FindShard(descriptor, key) != null; + + // Whether the database serves this cached instance: a shard owns its schema and key. A shard that owns everything makes + // the lookup of the instance's key unnecessary. + private bool Serves(object document) { - return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0]; + if (_shards.Any(shard => shard.SchemaIds.Count == 0 && string.IsNullOrEmpty(shard.KeyPrefix))) + { + return true; + } + + var type = document.GetType(); + return Owns(_cache.Registry.GetRequired(type), GetTrackedKey(document, type)); } - private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) - { - return _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) ?? _shards[0]; - } + private CultNetShardDescriptor? FindShard(CultDocumentDescriptor descriptor, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)); private static void EnsurePrimary(CultNetShardDescriptor shard, string schemaId, CultRecordKey key, long? expectedEpoch = null) { diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index d53315ed..77b03070 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -112,10 +112,13 @@ namespace GameCult.Networking request); } + // A database serves only what its shards own. return _database.Documents.CreateRawSnapshotResponse( _database.Cache, string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId, - request); + request, + options: null, + rowFilter: _database.Owns); } /// @@ -135,6 +138,9 @@ namespace GameCult.Networking // R-Q: asOf is one shard-log watermark, so a selection whose matched rows span more than // one shard's log is refused rather than answered against a watermark that is not // exact for all of them. + // A row whose schema no shard owns is not part of what this database serves: it is left out, not an error. + options: null, + rowFilter: _database.Owns, shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId, cursorKey: _database.CursorKey, // S-9: once the matched rows resolve to one shard, asOf is that shard's own watermark, @@ -312,6 +318,16 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + // The write committed and was published; only its shard log failed, so the peer is not told it was refused. + _server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { _server.Logger.LogError($"CultNet raw put failed: {ex.Message}"); @@ -332,6 +348,15 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + _server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); @@ -604,6 +629,12 @@ namespace GameCult.Networking : messageId; } + // A CultNetShardLogException means the write committed and was published (the writer hears of a failed log afterwards); + // the cache wraps it in an AggregateException when something else failed in the same hold. + private static bool CommittedWithLogFailure(Exception exception) => + exception is CultNetShardLogException || + exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException); + private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception) { return new CultNetErrorMessage diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index f3150740..7ae3c67e 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -421,7 +421,8 @@ namespace GameCult.Networking documentSelection, ordinalOf: (schemaId, key) => _database.LastWriteSequence(schemaId, key) ?? 0, asOf: _database.CurrentAsOf(), - options: null); + options: null, + rowFilter: _database.Owns); // The snapshot is one row per record key by construction, so the duplicate-key throw this // used to guard can no longer fire (docs/cultnet-selection-cut.md, S2-2); deleted rather // than tested around. diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index eb23d503..62691a3e 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -280,7 +280,16 @@ namespace GameCult.Networking CultCache cache, string messageId, CultNetSnapshotRequestMessage? filter = null, - CultNetDocumentMessageOptions? options = null) + CultNetDocumentMessageOptions? options = null) => + CreateRawSnapshotResponse(cache, messageId, filter, options, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse( + CultCache cache, + string messageId, + CultNetSnapshotRequestMessage? filter, + CultNetDocumentMessageOptions? options, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); @@ -323,7 +332,7 @@ namespace GameCult.Networking ordinalOf = static (_, _) => 0; } - var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options); + var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options, rowFilter); return new CultNetSnapshotResponseRawMessage { @@ -348,13 +357,27 @@ namespace GameCult.Networking CultNetDocumentMessageOptions? options = null, Func? shardIdOf = null, CultNetSelectionCursorKey? cursorKey = null, - Func? asOfForShard = null) + Func? asOfForShard = null) => + CreateSelectionResponse(cache, messageId, selection, ordinalOf, asOf, options, shardIdOf, cursorKey, asOfForShard, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawV1Message CreateSelectionResponse( + CultCache cache, + string messageId, + CultNetSelection selection, + Func ordinalOf, + ulong asOf, + CultNetDocumentMessageOptions? options, + Func? shardIdOf, + CultNetSelectionCursorKey? cursorKey, + Func? asOfForShard, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (selection == null) throw new ArgumentNullException(nameof(selection)); if (ordinalOf == null) throw new ArgumentNullException(nameof(ordinalOf)); - var page = SelectPage(cache, selection, ordinalOf, asOf, options, shardIdOf: shardIdOf, cursorKey: cursorKey, asOfForShard: asOfForShard); + var page = SelectPage(cache, selection, ordinalOf, asOf, options, rowFilter, shardIdOf, cursorKey, asOfForShard); return new CultNetSnapshotResponseRawV1Message { MessageId = RequireNonEmpty(messageId, nameof(messageId)), diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 4daa76d5..e7f4cfd9 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -511,7 +511,7 @@ namespace GameCult.Networking [Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; } /// /// Gets or sets the machine-readable refusal code (R-N), e.g. selection_invalid, - /// cursor_stale, cursor_invalid, reference_outside_target. Additive to + /// cursor_stale, cursor_invalid, reference_outside_target, unowned_schema. Additive to /// cultnet.error.v0; a v0 peer that does not know this field ignores it. /// [Key("code")] public string? Code { get; set; } @@ -532,6 +532,17 @@ namespace GameCult.Networking Details = new CultNetErrorDetails { Field = ex.Field, Value = ex.Value } }; + /// + /// Builds the wire error for a write no shard owns: code unowned_schema, details { field: "schemaId", value }. + /// + public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) => + new CultNetErrorMessage + { + Error = $"unowned_schema: {ex.Message}", + Code = "unowned_schema", + Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId } + }; + /// /// Builds the wire error for a cursor refusal (R-N): code cursor_stale with details /// { asOf, current }, or code cursor_invalid with no details. diff --git a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs index 5b3a9c32..4c93e000 100644 --- a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs +++ b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs @@ -10,6 +10,11 @@ namespace GameCult.Networking /// /// Stores accepted shard-log entries in their replica catch-up wire shape. /// + /// + /// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of + /// the cache and blocks its readers too, because reads take the same gate. A store must not call the cache, or wait for a thread that does. Appends must throw rather than + /// swallow a failure: the database burns the sequence and compacts past it, and tells the writer. + /// public interface ICultNetShardMutationLogStore { /// diff --git a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs index acf5eb6c..63747c29 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs @@ -938,6 +938,32 @@ public sealed class CultMeshStreamingTests await act.Should().ThrowAsync(); } + // A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type + // out of the cache behind it. + [TestCase("tests.mesh_note.v1", true)] + [TestCase("tests.unrelated.v1", false)] + public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served) + { + var cache = new CultCache(); + var key = new CultRecordKey("note:one"); + await cache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cached" }, new CultRecordHandle(key)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "reader", + Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) } + }); + var handle = CultMesh.Document(database, key, CultMesh.Verse("starbridge", "reader")); + + if (served) + { + (await handle.LatestAsync()).Text.Should().Be("cached"); + } + else + { + await FluentActions.Awaiting(() => handle.LatestAsync()).Should().ThrowAsync(); + } + } + [Test] public async Task DocumentHandle_SubmitsPredictionsThroughCultNetDatabase() { diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs new file mode 100644 index 00000000..6285d674 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -0,0 +1,1199 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // The database's shard log is written by a cache journal, under the cache gate, in cache order. A primary that + // cannot log a committed change burns that change's sequence and compacts past it, so the log can lag the cache but + // never lead it, and every replica behind the hole resynchronizes by snapshot. + public sealed class CultNetJournalTests + { + private const string ShardId = "log-shard"; + + private static readonly CultRecordKey One = new("tests:journal:one"); + private static readonly CultRecordKey Two = new("tests:journal:two"); + private static readonly CultRecordKey Three = new("tests:journal:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetShardDescriptor Shard(CultCache cache, bool primary) => + new(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)]); + + private static CultNetDatabase Database(CultCache cache, bool primary, FlakyLogStore? store) => + new(cache, new CultNetDatabaseOptions { Shards = [Shard(cache, primary)], MutationLogStore = store }); + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + private static List Record(CultNetDatabase database) + { + var seen = new List(); + database.WatchAllChanges().Subscribe(change => seen.Add(KeyOfChange(change))); + return seen; + } + + private static long[] Sequences(FlakyLogStore store) => store.Read(ShardId).Select(entry => entry.Sequence).ToArray(); + + // A store whose refusals and side effects the test scripts. One shard per store. + private sealed class FlakyLogStore : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compactedThrough; + + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public Action? OnRead { get; set; } + public Action? OnAppend { get; set; } + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + OnRead?.Invoke(); + lock (_gate) + { + var entries = _entries.Values.Where(entry => entry.Sequence > afterSequence); + return (limit.HasValue ? entries.Take(limit.Value) : entries).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + OnAppend?.Invoke(); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) + throw new IOException($"The log refused sequence {entry.Sequence}."); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) + { + lock (_gate) + return _compactedThrough; + } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) + throw new IOException("The log refused to compact."); + _compactedThrough = Math.Max(_compactedThrough, sequence); + foreach (var stale in _entries.Keys.Where(key => key <= sequence).ToArray()) + _entries.Remove(stale); + } + } + } + + // The primary as a replica meets it: the database server's own log answer, and the database's own snapshot. + private sealed class PrimaryPeer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _databaseServer; + private readonly CultNetDatabase _database; + + public PrimaryPeer(CultCache cache, CultNetDatabase database) + { + _database = database; + _server = new Server(cache, ServerSecurityOptions.Development()); + _databaseServer = new CultNetDatabaseServer(_server, database); + } + + public CultNetShardLogResponseMessage Log(long afterSequence) => + _databaseServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { + MessageId = "pull", + ShardId = ShardId, + ShardEpoch = 1, + AfterSequence = afterSequence + }); + + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => + Task.FromResult(Log(afterSequence)); + + public Task FetchAsync(CultNetShardDescriptor shard) => + Task.FromResult(_database.CreateShardSnapshotResponse(_database.Shards[0], "snapshot")); + + public void Dispose() + { + _databaseServer.Dispose(); + _server.Dispose(); + } + } + + private static CultNetShardReplicator Replicator(CultNetDatabase replica, PrimaryPeer peer) => + new(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + + private static string? TextOf(CultCache cache, CultRecordKey key) => cache.Get(key)?.Text; + + [Test] + public async Task ARefusedPrimaryAppendPublishesTheChangeTheWriterHearsOfItAndAReplicaBehindItResyncs() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(1)); + var published = Record(primary); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 2) })); + Assert.That(failure, Is.Not.InstanceOf(), "a committed write must not read as a refused one"); + Assert.That(published, Is.EqualTo(new[] { Two.Value }), "the change was published before the writer heard"); + Assert.That(TextOf(cache, Two), Is.EqualTo("two"), "the commit stands"); + Assert.That(primary.LastWriteSequence(SchemaId(cache), Two), Is.EqualTo(2L), "the row was written at the burned sequence"); + await primary.PutAsync(Three, Note("three")); + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "sequence 2 stays a hole below the compaction point"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3), "a later pull is a quiet no-op"); + } + + [Test] + public async Task ARestartAfterARefusedTailAppendReusesNoSequence() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using (var before = new PrimaryPeer(cache, primary)) + { + await primary.PutAsync(One, Note("one")); + using var replicator = Replicator(replica, before); + await replicator.PullOnceAsync(ShardId); + } + + Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two"))); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "the burned sequence was not minted again"); + using var after = new PrimaryPeer(cache, restarted); + using var resumed = Replicator(replica, after); + await resumed.PullOnceAsync(ShardId); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test] + public async Task AStoreThatRefusesTheAppendAndTheCompactionStillCompactsInMemoryAndNamesBothFailures() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.InnerException, Is.InstanceOf()); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2)); + Assert.That(primary.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2), "the primary's own floor covers a hole its store never compacted"); + Assert.That(primary.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the floor"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two")); + } + + [Test] + public async Task CompactingAndRestartingResumesTheNextSequenceAboveTheCompactionPoint() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + store.CompactThrough(ShardId, 2); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the compaction point"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L })); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(3)); + } + + [Test] + public async Task ARefusedReplicaAppendRecordsNothingAdvancesNoCursorAndTheRetryConverges() + { + var refusals = 0; + var replicaStore = new FlakyLogStore { RefuseAppend = sequence => sequence == 1 && refusals++ == 0 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + using var replicator = Replicator(replica, peer); + + Assert.ThrowsAsync(async () => await replicator.PullOnceAsync(ShardId)); + + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(0)); + Assert.That(replica.GetMutationLog(ShardId), Is.Empty, "memory never holds what the durable log refused"); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L })); + } + + [Test] + public async Task AReentrantShardLogApplyFromAHandlerKeepsTheReplicaLogInPrimaryOrder() + { + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + await primary.PutAsync(Three, Note("three")); + var response = new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = primary.GetMutationLogMessages(ShardId).ToArray() + }; + var replicaStore = new FlakyLogStore(); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + var nested = false; + long nestedApplied = 0; + Exception? nestedFailure = null; + replica.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + { + if (nested) return; + nested = true; + try + { + nestedApplied = replica.ApplyShardLogResponseAsync(response).GetAwaiter().GetResult(); + } + catch (Exception exception) + { + nestedFailure = exception; + } + }); + + await replica.ApplyShardLogResponseAsync(response); + + Assert.That(nested, Is.True); + Assert.That(nestedFailure, Is.Null, "the nested apply re-records entries the outer apply already logged"); + Assert.That(nestedApplied, Is.EqualTo(3)); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + Assert.That(replica.CurrentAsOf(ShardId), Is.EqualTo(3UL), "the replica's own watermark follows the primary's sequences"); + } + + [Test] + public async Task AReentrantDatabaseWriteFromAHandlerIsLoggedWithItsContextAndPublishedOnce() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var message = database.Documents.CreateRawDocumentPutMessage( + "reentrant-two", new CultRecordHandle(Two), Note("two")); + var published = Record(database); + database.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + database.ApplyPutAsync(message).GetAwaiter().GetResult()); + + await database.PutAsync(One, Note("one")); + + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(store.Read(ShardId).Single(entry => entry.Sequence == 2).Put!.MessageId, Is.EqualTo("reentrant-two"), + "the nested write's own wire message is what the log stored"); + } + + [Test] + public async Task ALogStoreThatWritesTheCacheFailsThroughTheReentryGuardAndTheAdmissionStands() + { + var cache = new CultCache(); + var store = new FlakyLogStore(); + store.OnAppend = () => + { + store.OnAppend = null; + cache.UpsertAsync(Note("from the store"), new CultRecordHandle(Three)).GetAwaiter().GetResult(); + }; + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.ThrowsAsync(async () => await database.PutAsync(One, Note("one")))!; + + Assert.That(failure.InnerException, Is.InstanceOf(), "the cache's guard refused the store's write"); + Assert.That(TextOf(cache, One), Is.EqualTo("one")); + Assert.That(TextOf(cache, Three), Is.Null); + Assert.That(published, Is.EqualTo(new[] { One.Value })); + } + + [Test] + public void SeveralRefusedAppendsInOneAdmissionAreAllPublishedAndAllReachTheWriter() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 1), new CultNetBurnedSequence(ShardId, 2) }), "one exception carries every burned sequence"); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2), "and every store failure"); + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2)); + } + + [Test] + public async Task APullOnAPrimaryIsLoggedSoAReplicaConverges() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-pull-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("seed"), new CultRecordHandle(One)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var primary = Database(cache, primary: true, new FlakyLogStore()); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + + await writer.UpsertAsync(Note("pulled"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await replicator.PullOnceAsync(ShardId); + + Assert.That(primary.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Does.Contain(Two.Value)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("pulled")); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task AStoreAttachedAfterTheDatabaseExistsHasItsHydrationLogged() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-attach-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("one"), new CultRecordHandle(One)); + await writer.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore()); + + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EquivalentTo(new[] { One.Value, Two.Value })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ASnapshotReadsItsLogSequenceBeforeItsDocumentsSoARacingWriteConverges() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + store.OnRead = () => + { + store.OnRead = null; + primary.PutAsync(Two, Note("two")).GetAwaiter().GetResult(); + }; + + var snapshot = primary.CreateShardSnapshotResponse(primary.Shards[0], "snapshot"); + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], snapshot); + await replicator.PullOnceAsync(ShardId); + + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two"), "a write racing the snapshot reaches the replica by one road or the other"); + } + + [Test] + public async Task ASnapshotAppliedToAPrimaryShardLogsNothing() + { + var cache = new CultCache(); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(One)); + var database = Database(cache, primary: true, new FlakyLogStore()); + var put = database.Documents.CreateRawDocumentPutMessage("snapshot-two", new CultRecordHandle(Two), Note("two")); + + await database.ApplyShardSnapshotResponseAsync(database.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(TextOf(cache, One), Is.Null); + Assert.That(TextOf(cache, Two), Is.EqualTo("two")); + Assert.That(database.GetMutationLog(ShardId), Is.Empty, "the snapshot's removal and its put are both a resync, not a commit"); + } + + [Test] + public async Task ARestartOverARetainedLogResumesAboveItsHighestSequence() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "a restarted database reads its latest sequence from the durable log"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L, 2L, 3L })); + } + + [Test] + public async Task APutAndDeleteDoorLogTheirOwnWireMessagesAndChangeKinds() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var added = database.Documents.CreateRawDocumentPutMessage("put-added", new CultRecordHandle(One), Note("one")); + var updated = database.Documents.CreateRawDocumentPutMessage("put-updated", new CultRecordHandle(One), Note("uno")); + + await database.ApplyPutAsync(added); + await database.ApplyPutAsync(updated); + await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage { MessageId = "delete-one", SchemaId = SchemaId(cache), RecordKey = One.Value }); + + var entries = store.Read(ShardId); + Assert.That(entries.Select(entry => entry.ChangeKind), Is.EqualTo(new[] { "added", "updated", "removed" })); + Assert.That(entries[0].Put!.MessageId, Is.EqualTo("put-added")); + Assert.That(entries[1].Put!.MessageId, Is.EqualTo("put-updated")); + Assert.That(entries[2].Delete!.MessageId, Is.EqualTo("delete-one")); + Assert.That(DateTimeOffset.TryParse(entries[0].CommittedAt, out _), Is.True, "a primary stamps its entries"); + } + + [Test] + public async Task AReconciledPutIsLoggedAsTheUpdateItIs() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(One), Note("authoritative")); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + var published = new List(); + database.Watch().Subscribe(change => published.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("predicted")); + await database.ApplyPutAsync(message); + + Assert.That(published, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(database.GetMutationLog(shard.ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated }), + "a prediction is never logged and its reconciliation is an ordinary update"); + } + + [Test] + public async Task AnAdmissionRacingDisposalLogsNothingAfterTheDatabaseIsDisposed() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var entered = new System.Threading.ManualResetEventSlim(); + CultNetDatabase? database = null; + // Registered before the database, so it runs first inside the writer's hold, while another thread disposes the database. + using var holdUntilDisposed = cache.AddJournal(_ => + { + entered.Set(); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + try + { + database!.GetLatestMutationLogSequence(ShardId); + } + catch (ObjectDisposedException) + { + return; + } + + System.Threading.Thread.Sleep(1); + } + }); + database = Database(cache, primary: true, store); + var writer = Task.Run(() => database.PutAsync(One, Note("one"))); + Assert.That(entered.Wait(TimeSpan.FromSeconds(5)), Is.True); + + database.Dispose(); + await writer; + + Assert.That(Sequences(store), Is.Empty, "the admission was journaled after disposal began and must not reach the durable log"); + } + + [Test] + public void ABatchAcrossTwoShardsWhoseStoreRefusesReachesTheWriterAsOneExceptionNamingBothShards() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = + [ + new CultNetShardDescriptor("shard-a", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "a:"), + new CultNetShardDescriptor("shard-b", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "b:") + ], + MutationLogStore = store + }); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("a"), new CultRecordHandle(new CultRecordKey("a:1"))); + batch.Upsert(Note("b"), new CultRecordHandle(new CultRecordKey("b:1"))); + }))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence("shard-a", 1), new CultNetBurnedSequence("shard-b", 1) })); + } + + // The cache aggregates whatever else fails in the same hold; the documented unwrap finds the log failure inside. + [Test] + public void ALogFailureBesideAnotherFailureInTheSameHoldIsFoundByUnwrappingTheAggregate() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Database(cache, primary: true, store); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + var logFailure = failure.InnerExceptions.OfType().Single(); + Assert.That(logFailure.Burned.Select(burned => burned.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(failure.InnerExceptions.OfType().Single().Message, Is.EqualTo("second journal")); + } + + [Test] + public async Task ABareWriteOfASchemaNoShardOwnsIsNeitherLoggedNorPublishedNorReplicated() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var published = Record(database); + + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "n", Body = "unrelated" }, new CultRecordHandle(new CultRecordKey("unrelated:n"))); + await cache.UpsertAsync(Note("one"), new CultRecordHandle(One)); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the unrelated write minted no sequence"); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); + Assert.That(published, Is.EqualTo(new[] { One.Value }), "no shard owns the unrelated write, so the database does not publish it"); + } + + // The database is the sharded write door: a schema and key no shard owns cannot be written on any path, and a refused + // write caches and logs nothing. + private static (CultNetDatabase Database, CultCache Cache, FlakyLogStore Store, string SchemaId) OwnedNothingOfNote() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + return (Database(cache, primary: true, store), cache, store, cache.Registry.GetRequired().SchemaId); + } + + private static readonly CultRecordKey Unowned = new("unrelated:note"); + + private static CultNetDocumentPutRawMessage UnownedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "wire" }); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + [Test] + public void ADatabasePutOrDeleteOfASchemaNoShardOwnsIsRefusedNamingTheSchema() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + + var put = Assert.ThrowsAsync(async () => + await database.PutAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "plain" }))!; + Assert.That(put.SchemaId, Is.EqualTo(unowned)); + Assert.That(put.Message, Does.Contain(unowned)); + Assert.That(put, Is.InstanceOf(), "a refused write reads as one"); + Assert.That(cache.Get(Unowned), Is.Null, "nothing was cached"); + Assert.ThrowsAsync(async () => await database.DeleteAsync(Unowned)); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public void ARemoteAuthoritativePutOfASchemaNoShardOwnsIsRefusedAndCachesNothing() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(UnownedPut(database)))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Null); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public async Task ARemoteDeleteOfASchemaNoShardOwnsIsRefusedEvenWhenTheKeyIsCached() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + + var refusal = Assert.ThrowsAsync(async () => await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + }))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the refused delete removed nothing"); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public void APredictionOfASchemaNoShardOwnsIsRefused() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + Shards = [new CultNetShardDescriptor("only-notes", "local", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)])], + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + + Assert.ThrowsAsync(async () => + await database.PutPredictedAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "predicted" })); + Assert.That(cache.Get(Unowned), Is.Null); + } + + [Test] + public void ASchemaNoShardOwnsCannotBeWrittenAuthoritativelyAndHasNoShardToResolve() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + + Assert.That(database.CanWriteAuthoritatively(Unowned), Is.False); + Assert.That(database.CanWriteAuthoritatively(One), Is.True, "an owned schema is unaffected"); + var refusal = Assert.Throws(() => database.ResolveShard(unowned, Unowned))!; + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(database.ResolveShard(SchemaId(cache), One).ShardId, Is.EqualTo(ShardId)); + } + + // The server answers a refused remote put the way it answers any other: it logs the failure and sends the peer a + // CultNetErrorMessage carrying the refusal. The handler is reached through its private delegate with an + // uninitialized peer, as the R-AM test does; the send then fails on that peer, after the failure was logged. + [Test] + public void TheServerAnswersARemotePutOfASchemaNoShardOwnsWithAnApplicationRejection() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await handler(UnownedPut(database), peer)); + + Assert.That(logger.Warnings, Has.Some.Contains("raw put refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty, "a refusal is not a fault"); + Assert.That(cache.Get(Unowned), Is.Null); + } + + // One cached row of a schema no shard owns is not part of what this database serves: the selection answers with the + // rest, it does not fail as a whole. + [Test] + public async Task ASelectionSkipsACachedRowNoShardOwnsAndAnswersWithTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSelectionResponse(new CultNetSnapshotRequestV1Message + { + MessageId = "select-all", + Selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document } + }); + + Assert.That(response.Documents!.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + Assert.That(response.Matched, Is.EqualTo(1u)); + } + + [CultDocument("tests.journal_indexed", "tests.journal_indexed.v1")] + [MessagePack.MessagePackObject] + public sealed class JournalIndexedNote + { + [MessagePack.Key(0)] + [CultName] + public string Name { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultIndex("tag")] + public string Tag { get; set; } = string.Empty; + } + + // A database serves only what its shards own: reads, the legacy and shard-less snapshot, selections, subscription + // snapshots and the watch agree. The cache behind it still holds the row. + [Test] + public async Task ADatabaseReadsServeOnlyWhatItsShardsOwnWhileTheCacheStillHoldsTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + var seen = Record(database); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new JournalIndexedNote { Name = "n", Tag = "t" }, new CultRecordHandle(new CultRecordKey("unrelated:indexed"))); + + Assert.That(database.GetAll(), Is.Empty); + Assert.That(database.GetAll().Count(), Is.EqualTo(1), "only the owned row"); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(await database.GetAsync(Unowned), Is.Null); + Assert.That(await database.GetAsync(One), Is.Not.Null); + Assert.That(database.GetByName("u"), Is.Null); + Assert.That(database.GetByName("n"), Is.Null); + Assert.That(database.GetByIndex("tag", "t"), Is.Null); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the cache itself still holds it"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "what the database reads out is what it publishes"); + } + + [Test] + public async Task ADatabaseCollectionOfASchemaNoShardOwnsIsEmpty() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + + var collection = GameCult.Mesh.CultMesh.Collection(database); + + Assert.That(await collection.LatestAsync(), Is.Empty); + } + + [Test] + public async Task TheShardlessSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + + Assert.That(response.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + + [Test] + public async Task ASubscriptionSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + var socket = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Dgram, System.Net.Sockets.ProtocolType.Udp); + socket.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "owned-reads-server", Socket = socket }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("owned-reads-client"); + CultNetSnapshotResponseRawMessage? snapshot = null; + client.OnCultNet(message => snapshot = message); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + for (var attempt = 0; attempt < 300 && !client.Connected; attempt++) + await Task.Delay(10); + client.SendCultNet(new CultNetDatabaseSubscribeMessage { MessageId = "subscribe-all", SubscriptionId = "all", IncludeSnapshot = true }); + for (var attempt = 0; attempt < 300 && snapshot == null; attempt++) + await Task.Delay(10); + + Assert.That(snapshot, Is.Not.Null, "the subscription answered a snapshot"); + Assert.That(snapshot!.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + + // A replica applies the log the way it applies a snapshot: a row its shard does not own is left out. The entry is + // still recorded, so a replica chained on this one sees no gap. + [Test] + public async Task AReplicaLogApplySkipsARowItsShardDoesNotOwnButRecordsTheEntry() + { + var cache = new CultCache(); + var replica = Database(cache, primary: false, store: null); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey("unrelated:put")), new MeshQuickstartNote { NoteId = "p", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + var owned = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("owned")); + owned.ShardId = ShardId; + owned.ShardEpoch = 1; + var now = DateTimeOffset.UtcNow.ToString("O"); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = now, ChangeKind = "added", Put = put }, + new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = cache.Registry.GetRequired().SchemaId, + RecordKey = Unowned.Value, + ShardId = ShardId, + ShardEpoch = 1 + } + }, + new CultNetShardLogEntryMessage { Sequence = 3, CommittedAt = now, ChangeKind = "added", Put = owned } + ] + }); + + Assert.That(cache.Get(new CultRecordKey("unrelated:put")), Is.Null, "the put was left out"); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the delete removed nothing the shard owns"); + Assert.That(TextOf(cache, One), Is.EqualTo("owned"), "owned entries still apply"); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + } + + // The server's put and delete handlers are reached through their private delegates with an uninitialized peer (as + // the R-AM test does): anything the handler tries to send throws on that peer, so a handler that returns quietly + // sent nothing. + private static (CultNetDatabaseServer Server, Func Put, Func Delete, CapturingLogger Logger, Server Host) WireHandlers(CultNetDatabase database, CultCache cache) + { + var host = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + host.Logger = logger; + var databaseServer = new CultNetDatabaseServer(host, database); + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + const System.Reflection.BindingFlags flags = System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance; + var put = (Func)typeof(CultNetDatabaseServer).GetField("_putHandler", flags)!.GetValue(databaseServer)!; + var delete = (Func)typeof(CultNetDatabaseServer).GetField("_deleteHandler", flags)!.GetValue(databaseServer)!; + return (databaseServer, message => put(message, peer), message => delete(message, peer), logger, host); + } + + private static CultNetDocumentPutRawMessage OwnedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + return message; + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedBesideAnotherFailureIsStillNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire")); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemoteDeleteWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }); + await database.PutAsync(One, Note("one")); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = SchemaId(cache), + RecordKey = One.Value, + ShardId = ShardId, + ShardEpoch = 1 + }); + + Assert.That(TextOf(cache, One), Is.Null, "the delete committed"); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public void ARemoteDeleteOfASchemaNoShardOwnsIsRefusedOnTheWireAtWarningLevel() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + })); + + Assert.That(logger.Warnings, Has.Some.Contains("raw delete refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty); + } + + // A change the log holds is published, however Dispose interleaves with the write: Dispose waits for a journal in + // flight (disposing the journal first) and publishes whatever the writer had logged and not yet published. The writer + // is held in an earlier cache observer, so Dispose finishes before the writer reaches the database's own observer. + [Test] + public async Task ADatabaseDisposedWhileAWriteIsInItsLogAppendStillPublishesWhatItLogged() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + using var inObserver = new ManualResetEventSlim(); + using var releaseObserver = new ManualResetEventSlim(); + using var holdWriter = cache.Watch().Subscribe(_ => + { + inObserver.Set(); + releaseObserver.Wait(TimeSpan.FromSeconds(10)); + }); + var database = Database(cache, primary: true, store); + var seen = Record(database); + using var inAppend = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + store.OnAppend = () => + { + inAppend.Set(); + release.Wait(TimeSpan.FromSeconds(10)); + }; + var writer = Task.Run(async () => await database.PutAsync(One, Note("one"))); + Assert.That(inAppend.Wait(TimeSpan.FromSeconds(10)), Is.True); + + var dispose = Task.Run(database.Dispose); + await Task.Delay(300); + release.Set(); + Assert.That(inObserver.Wait(TimeSpan.FromSeconds(10)), Is.True, "the writer left the gate and is held in the earlier observer"); + await dispose; + releaseObserver.Set(); + await writer; + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the write was logged"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "and published"); + } + + private sealed class CapturingLogger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public List Warnings { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) => Warnings.Add(message); + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static async Task ApplyAuthoritativeAsync(CultNetDatabase database, CultRecordKey key, string text) + { + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(key), Note(text)); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + await database.ApplyPutAsync(message); + } + + private static CultNetDatabaseOptions ClientOptions() => new() + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }; + + [Test] + public async Task APredictionOverAPredictionIsStillAPredictionAndTheAuthoritativeWriteReconcilesOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, ClientOptions()); + var kinds = new List(); + database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("p1")); + await database.PutPredictedAsync(One, Note("p2")); + await ApplyAuthoritativeAsync(database, One, "authoritative"); + await ApplyAuthoritativeAsync(database, One, "again"); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + [Test] + public async Task AGameSessionPredictionOfTheInstanceTheCacheAlreadyHoldsIsAPredictionThatIsNeverLogged() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "journal-session", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + using var node = await GameCult.Mesh.CultMesh.CreateNodeAsync(Path.Combine(rootPath, "world.ccmp"), new GameCult.Mesh.CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = ClientOptions() + }); + using var session = GameCult.Mesh.CultMesh.CreateGameSession(node, new GameCult.Mesh.CultMeshGameSessionOptions + { + ServeSimulationObservations = false, + ServeVerseDiscovery = false, + ServePeerExchange = false + }); + var kinds = new List(); + node.Database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await session.PredictAsync(One, Note("p1")); + var held = node.Cache.Get(One)!; + held.Text = "p2"; + await session.PredictAsync(One, held); + await ApplyAuthoritativeAsync(node.Database, One, "authoritative"); + + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(node.Database.GetMutationLog(node.Database.Shards[0].ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + [Test] + public async Task ADatabaseRecordsEveryDeleteEntryItAppliesWhileAnotherThreadRemovesTheSameKeys() + { + const int rounds = 20000; + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + var schemaId = SchemaId(replicaCache); + for (var i = 1; i <= rounds; i++) + await replicaCache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey("race:" + i))); + + var current = 0; + var stop = false; + var remover = Task.Run(() => + { + var next = 1; + while (next <= rounds && !Volatile.Read(ref stop)) + { + if (Volatile.Read(ref current) >= next) + { + replicaCache.Remove(new CultRecordKey("race:" + next)); + next++; + } + } + }); + for (var i = 1; i <= rounds; i++) + { + Volatile.Write(ref current, i); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage + { + Sequence = i, + CommittedAt = DateTimeOffset.UtcNow.ToString("O"), + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d" + i, + SchemaId = schemaId, + RecordKey = "race:" + i, + ShardId = ShardId, + ShardEpoch = 1 + } + } + ] + }); + } + + Volatile.Write(ref stop, true); + await remover; + + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(Enumerable.Range(1, rounds).Select(i => (long)i)), + "a replica's log holds every entry of the primary's, whoever removed the key first"); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs new file mode 100644 index 00000000..62d31e69 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs @@ -0,0 +1,351 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + public sealed class CultNetPublicationOrderTests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text; + + // A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed + // (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile. + [Test] + public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "A") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache); + var seen = new List(); + using var subscriber = database.Watch().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); }); + var key = new CultRecordKey("soul:order"); + + var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle(key)).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True); + await database.PutAsync(key, Note("B")); + release.Set(); + await a; + + var shard = database.Shards[0].ShardId; + var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last(); + + // A replica built from the log. + var replicaCache = new CultCache(); + var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)] + }); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shard, + ShardEpoch = database.Shards[0].Epoch, + Entries = database.GetMutationLogMessages(shard).ToArray() + }); + + Assert.Multiple(() => + { + Assert.That(cache.Get(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B"); + Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write"); + Assert.That(replicaCache.Get(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary"); + // Delivery is not ordered across threads (order is data: a wire peer drops the stale change by Sequence, Cut 6), + // so a subscriber sees both changes, once each, in either order. + Assert.That(seen, Is.EquivalentTo(new[] { "A", "B" }), "each change is published exactly once"); + }); + } + + // Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache. + [Test] + public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing() + { + var divergent = 0; + const int rounds = 300; + for (var round = 0; round < rounds; round++) + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:hammer"); + using var start = new Barrier(2); + Parallel.For(0, 2, writer => + { + start.SignalAndWait(); + for (var i = 0; i < 40; i++) + cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle(key)).GetAwaiter().GetResult(); + }); + var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last(); + if (!ReferenceEquals(last.Document, cache.Get(key))) + divergent++; + } + + TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}"); + Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache"); + } + + // A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in + // the window before the door's own change is handled, consumes the door's context. + [Test] + public async Task S2_ARemovalNeverWearsAReconcilingPutsContext() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:reconcile"); + await database.PutPredictedAsync(key, Note("predicted")); + var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>(); + using var subscriber = database.Watch().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text))); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + var a = Task.Run(() => database.ApplyPutAsync(message)); + Assert.That(entered.Wait(Wait), Is.True); + cache.Remove(key); + release.Set(); + await a; + + var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray(); + Assert.Multiple(() => + { + Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1), + "the removal is published as a removal: " + string.Join(", ", published)); + Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text), + Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document"); + Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True, + "no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document)))); + }); + } + + // A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not + // hold (B removed it locally, a supported write) must still keep B's log contiguous for C. + [Test] + public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey() + { + const string shardId = "chain"; + var schemaId = new CultCache().Registry.GetRequired().SchemaId; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])] + }); + var p = Make(new CultCache(), true); + var bCache = new CultCache(); + var b = Make(bCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray() + }); + } + + var k = new CultRecordKey("soul:chain:k"); + await p.PutAsync(k, Note("x")); + await Pull(p, b); + await Pull(b, c); + bCache.Remove(k); + await p.DeleteAsync(k); + await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y")); + await Pull(p, b); + + Assert.Multiple(() => + { + Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }), + "B's log holds every primary entry it applied"); + Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap"); + }); + } + + // A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction. + [Test] + public async Task S4_APredictionMadeInReactionToAReconcileSurvives() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:repredict"); + var kinds = new ConcurrentQueue(); + var repredicted = false; + using var subscriber = database.Watch().Subscribe(change => + { + kinds.Enqueue(change.Kind); + if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted) + { + repredicted = true; + database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult(); + } + }); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled + })); + } + // Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A + // refused prediction leaves nothing behind that a later write of the same instance would wear. + [Test] + public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var baseKey = new CultRecordKey("soul-refuse-base"); + var variantKey = new CultRecordKey("soul-refuse-variant"); + await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 }, + new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v") + }); + var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 }; + Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException); + var kinds = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => kinds.Enqueue(change.Kind)); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await cache.UpsertAsync(instance, new CultRecordHandle(new CultRecordKey("soul-refuse-other"))); + + Assert.Multiple(() => + { + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1)); + }); + } + + // Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a + // durable log store can observe even though the disposed Subject hides publication. + [Test] + public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog() + { + var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N")); + var store = new CultNetFileShardMutationLogStore(root); + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store }); + var shard = database.Shards[0].ShardId; + await cache.UpsertAsync(Note("before"), new CultRecordHandle(new CultRecordKey("soul:dispose:before"))); + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + database.Dispose(); + + try { await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("soul:dispose:after"))); } + catch { /* a throwing handler would also be a failure below */ } + + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + System.IO.Directory.Delete(root, true); + } + + // The log entry exists before its change is published: a subscriber that reads the log from its handler finds it. + [Test] + public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:log-first"); + var loggedWhenPublished = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => + loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value))); + + await database.PutAsync(key, Note("one")); + cache.Remove(key); + + Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true })); + } + + // An observer that mutates the admitted instance in place and upserts it again writes a second change of the same + // instance. It is published as its own update and never wears the context of the door's change, even when it + // reaches the database first. + [Test] + public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext() + { + var cache = new CultCache(); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + change.Document.Text = "normalized"; + cache.UpsertAsync(change.Document, new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:in-place"); + var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>(); + using var subscriber = database.Watch().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text))); + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(message); + + // The observer's nested write is delivered depth-first, inside the delivery of the change that caused it, so + // the database may publish the two in either order; what is pinned is that each carries its own context. + Assert.That(seen, Is.EquivalentTo(new (CultNetDatabaseChangeKind, string?)[] + { + (CultNetDatabaseChangeKind.Predicted, null), + (CultNetDatabaseChangeKind.Reconciled, "predicted"), + (CultNetDatabaseChangeKind.Updated, "normalized") + })); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs new file mode 100644 index 00000000..fbd1df2a --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs @@ -0,0 +1,429 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; +using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests; + +namespace GameCult.Networking.Tests +{ + // The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the + // mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every + // other change in the hold publishes from the cache. + public sealed class CultNetPublishesCommitsTests + { + private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) => + wire.Changes.Where(change => KeyOf(change) == key).ToArray(); + + private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey; + + private sealed class Wire : IDisposable + { + private readonly CancellationTokenSource _cancellation = new(); + private readonly Thread _thread; + private readonly RudpCultNetSchemaServer _server; + private readonly CultNetDatabaseSubscriptionServer _subscriptions; + private readonly ICultNetSchemaClient _client; + public ConcurrentQueue Changes { get; } = new(); + public int Snapshots; + public ConcurrentQueue Errors { get; } = new(); + + private readonly CultNetDatabase _database; + + public Wire(CultNetDatabase database) + { + _database = database; + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + _server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket }); + _subscriptions = new CultNetDatabaseSubscriptionServer(_server, database); + _thread = new Thread(() => + { + while (!_cancellation.IsCancellationRequested) + { + _ = _server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + _thread.Start(); + _client = CultNetSchemaClients.CreateRudp("publish-client"); + _client.OnCultNet(Changes.Enqueue); + _client.OnCultNet(Errors.Enqueue); + _client.OnCultNet(_ => Interlocked.Increment(ref Snapshots)); + _client.Connect("127.0.0.1", _server.LocalEndPoint.Port); + } + + public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes) + { + await Until(() => _client.Connected); + _client.SendCultNet(new CultNetDatabaseSubscribeMessage + { + MessageId = "subscribe", + SubscriptionId = "all", + SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(), + IncludeSnapshot = includeSnapshot + }); + await Until(() => Snapshots > 0 || !Errors.IsEmpty); + Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused"); + } + + public void Dispose() + { + _cancellation.Cancel(); + _thread.Join(); + _client.Dispose(); + _subscriptions.Dispose(); + _server.Dispose(); + } + } + + private static async Task Until(Func condition) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + Assert.Fail("Condition was not satisfied before timeout."); + await Task.Delay(5); + } + } + + // One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance + // to arrive twice. + private static async Task SettleAsync(CultNetDatabase database, Wire wire) + { + var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N"); + await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" }); + await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel)); + } + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + [Test] + public async Task ACacheCommitAndUpsertReachASubscriberOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:commit"); + + cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle(key))); + await Until(() => Of(wire, key.Value).Length >= 1); + cache.Commit(batch => batch.Remove(key)); + await Until(() => Of(wire, key.Value).Length >= 2); + await cache.UpsertAsync(Note("upserted"), new CultRecordHandle(key)); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3)); + } + + [Test] + public async Task ADatabasePutAndDeleteStillPublishExactlyOnce() + { + var database = new CultNetDatabase(new CultCache()); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:put"); + + await database.PutAsync(key, Note("put")); + await database.DeleteAsync(key); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + [Test] + public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:replicated"); + var message = database.Documents.CreateRawDocumentPutMessage( + "put-replicated", new CultRecordHandle(key), Note("remote")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.ApplyPutAsync(message); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1)); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1)); + } + + [Test] + public async Task ALoadRemovalCarriesItsKey() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc"); + try + { + var key = new CultRecordKey("tests:publish:loaded"); + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("there"), new CultRecordHandle(key)); + writer.FlushAllBackingStores(); + + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var database = new CultNetDatabase(cache); + Assert.That(cache.Get(key), Is.Not.Null); + using var wire = new Wire(database); + await wire.SubscribeAsync(true, typeof(NetworkSchemaNote)); + + writer.Remove(key); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big"), + cache.Override(nameof(VariantWireFixture.Power), 9) + }); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(VariantWireFixture)); + var errorsBefore = wire.Errors.Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + await Until(() => wire.Errors.Count > errorsBefore); + await Until(() => Of(wire, baseKey.Value).Length >= 1); + + Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value), + "the re-resolved dependent is published, then refused (Q6) by key"); + Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1)); + Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record"); + } + + // An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own. + [Test] + public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:normalized"); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "raw") + cache.UpsertAsync(Note("normalized"), new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + }); + + await database.PutAsync(key, Note("raw")); + await Until(() => Of(wire, key.Value).Length >= 2); + await SettleAsync(database, wire); + + Assert.That(cache.Get(key)!.Text, Is.EqualTo("normalized")); + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + return (cache, documents); + } + + private static System.Collections.Generic.List Record(CultNetDatabase database, Func describe) + { + var seen = new System.Collections.Generic.List(); + database.WatchAllChanges().Subscribe(change => seen.Add(describe(change))); + return seen; + } + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + [Test] + public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder() + { + var (cache, documents) = VariantCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big") + }); + var published = Record(database, KeyOfChange); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value), + Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + } + + private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) => + new(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + Shards = + [ + new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId]) + ] + }); + + [Test] + public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally() + { + var (sourceCache, sourceDocuments) = VariantCache(); + var (replicaCache, replicaDocuments) = VariantCache(); + var schemaId = sourceCache.Registry.GetRequired().SchemaId; + var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions + { + DocumentRegistry = sourceDocuments, + Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])] + }); + var replica = Replica(replicaCache, replicaDocuments, schemaId); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[] + { + replicaCache.Override(nameof(VariantWireFixture.Name), "big") + }); + await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Entries = source.GetMutationLogMessages("replica-shard").ToArray() + }); + + Assert.That(replicaCache.Get(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved"); + Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }), + "the replica log holds the primary's entry and mints none of its own"); + } + + [Test] + public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var published = Record(replica, KeyOfChange); + + await cache.UpsertAsync(Note("local"), new CultRecordHandle(new CultRecordKey("tests:publish:local"))); + + Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + const int writers = 4, perWriter = 5000; + + Parallel.For(0, writers, writer => + { + for (var i = 0; i < perWriter; i++) + cache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey($"tests:publish:c:{writer}:{i}"))) + .GetAwaiter().GetResult(); + }); + + var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray(); + Assert.That(sequences, Has.Length.EqualTo(writers * perWriter)); + Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter)); + } + + [Test] + public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("tests:publish:after-delete"); + await database.PutAsync(key, Note("one")); + await database.DeleteAsync(key); + var published = Record(database, KeyOfChange); + + await cache.UpsertAsync(Note("two"), new CultRecordHandle(key)); + + Assert.That(published, Is.EqualTo(new[] { key.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind), + Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added + })); + } + + [Test] + public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var stale = new CultRecordKey("tests:publish:stale"); + var fresh = new CultRecordKey("tests:publish:fresh"); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(stale)); + var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value); + var put = replica.Documents.CreateRawDocumentPutMessage( + "snapshot-fresh", new CultRecordHandle(fresh), Note("fresh")); + + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public async Task ADisposedDatabaseNoLongerObservesTheCache() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var published = Record(database, KeyOfChange); + database.Dispose(); + + await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("tests:publish:disposed"))); + + Assert.That(published, Is.Empty); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index b5f4f08f..3aa2705d 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -195,6 +195,17 @@ namespace GameCult.Networking.Tests AssertErrorMatchesSchema(CultNetErrorMessage.ForReferenceOutsideTarget(ex)); } + [Test] + public void ErrorMessage_UnownedSchemaRealBytes_MatchesItsSchema() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + + Assert.That(message.Code, Is.EqualTo("unowned_schema")); + Assert.That(message.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(message.Details.Value, Is.EqualTo("some.schema.v1")); + AssertErrorMatchesSchema(message); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index e32ca91a..1c849fd7 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1898,7 +1898,7 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(batched), Is.Null); }), Is.True); - Assert.That(published, Is.EqualTo(new[] { put })); + Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it"); Assert.That(cache.Get(batched)?.Text, Is.EqualTo("batched")); var reopened = new CultCache(); reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); @@ -2627,7 +2627,8 @@ namespace GameCult.Networking.Tests var cache = new CultCache(); var delivering = new ManualResetEventSlim(); - cache.OnUpdate += (_, _) => delivering.Set(); + // Subscribed before the database, so it runs before the database's own observer of the same change. + using var loadDelivering = cache.Watch().Subscribe(_ => delivering.Set()); var database = new CultNetDatabase(cache); var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { @@ -4211,13 +4212,13 @@ namespace GameCult.Networking.Tests Schema = "tests.networking_note.v1", Text = "two" }); - // Outside the shard's key prefix: must reach the non-shard snapshot but never the + // Outside the shard's key prefix (written to the cache directly: the database refuses a write no shard owns): must reach the non-shard snapshot but never the // shard-scoped one, so an unfiltered request still proves shard membership is enforced. - await database.PutAsync(new CultRecordKey("other-note:outside-shard"), new NetworkSchemaNote + await cache.UpsertAsync(new NetworkSchemaNote { Schema = "tests.networking_note.v1", Text = "outside" - }); + }, new CultRecordHandle(new CultRecordKey("other-note:outside-shard"))); var nonShard = registry.CreateRawSnapshotResponse(cache, "snapshot-nonshard-unfiltered", filter: null); var shardScoped = database.CreateShardSnapshotResponse(shard, "snapshot-shard-unfiltered", filter: null); diff --git a/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs new file mode 100644 index 00000000..feb2cb3e --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass on Cut 4 (scratch; not for merge). Category Finding asserts the invariant Soul expects and is expected to + // fail on 13072fc5; Pin asserts a promise that should hold; Measure reports numbers. + public sealed class SoulP4ProbeTests + { + private const string ShardId = "soul-shard"; + private static readonly CultRecordKey One = new("soul:p4:one"); + private static readonly CultRecordKey Two = new("soul:p4:two"); + private static readonly CultRecordKey Three = new("soul:p4:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + private sealed class Store : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compacted; + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public int AppendDelayMs { get; set; } + public ManualResetEventSlim InAppend { get; } = new(); + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + lock (_gate) + { + var e = _entries.Values.Where(x => x.Sequence > afterSequence); + return (limit.HasValue ? e.Take(limit.Value) : e).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + InAppend.Set(); + if (AppendDelayMs > 0) Thread.Sleep(AppendDelayMs); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) throw new IOException($"refused {entry.Sequence}"); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) { lock (_gate) return _compacted; } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) throw new IOException("refused compact"); + _compacted = Math.Max(_compacted, sequence); + foreach (var k in _entries.Keys.Where(k => k <= sequence).ToArray()) _entries.Remove(k); + } + } + } + + private sealed class Peer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _dbServer; + private readonly CultNetDatabase _db; + public Peer(CultCache cache, CultNetDatabase db) + { + _db = db; + _server = new Server(cache, ServerSecurityOptions.Development()); + _dbServer = new CultNetDatabaseServer(_server, db); + } + public CultNetShardLogResponseMessage Log(long after) => _dbServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { MessageId = "pull", ShardId = ShardId, ShardEpoch = 1, AfterSequence = after }); + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => Task.FromResult(Log(afterSequence)); + public Task FetchAsync(CultNetShardDescriptor shard) => Task.FromResult(_db.CreateShardSnapshotResponse(_db.Shards[0], "snap")); + public void Dispose() { _dbServer.Dispose(); _server.Dispose(); } + } + + private static int StashCount(CultNetDatabase db) + { + var field = typeof(CultNetDatabase).GetField("_stash", BindingFlags.NonPublic | BindingFlags.Instance)!; + var stash = (System.Collections.ICollection)field.GetValue(db)!; + lock (typeof(CultNetDatabase).GetField("_logGate", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(db)!) + return stash.Count; + } + + // F1: the recorded residual is "tail reuse". A hole in the middle of the log after a double refusal and a restart + // strands a replica behind it for good: the restarted primary has no floor, the store serves S+1 after S-1. + [Test, Category("SoulFinding")] + public async Task F1_MidLogHoleAfterDoubleRefusalAndRestartStrandsAReplica() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + await primary.PutAsync(One, Note("one")); + using (var p0 = new Peer(cache, primary)) + using (var r0 = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = p0, SnapshotFetcher = p0 })) + await r0.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + store.RefuseCompact = false; + await primary.PutAsync(Three, Note("three")); + primary.Dispose(); + + var restarted = Db(cache, true, store); + using var peer = new Peer(cache, restarted); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + var errors = new List(); + for (var attempt = 0; attempt < 3; attempt++) + { + try { await replicator.PullOnceAsync(ShardId); } + catch (Exception e) { errors.Add(e.GetType().Name + ": " + e.Message); } + } + + TestContext.Out.WriteLine($"store={string.Join(",", store.Read(ShardId).Select(e => e.Sequence))} floor={restarted.GetCompactedMutationLogSequence(ShardId)} errors=[{string.Join(" | ", errors)}]"); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + // F2: Q-P5 A says the writer gets a typed CultNetShardLogException. A commit of two changes on a refusing store + // hands the writer an AggregateException; a writer's catch (CultNetShardLogException) misses it. + [Test, Category("SoulFinding")] + public void F2_ABatchOnARefusingStoreReachesTheWriterAsTheTypedException() + { + var store = new Store { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Db(cache, true, store); + Exception? caught = null; + try + { + cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }); + } + catch (Exception e) { caught = e; } + TestContext.Out.WriteLine($"writer caught {caught?.GetType().FullName}"); + Assert.That(caught, Is.InstanceOf()); + } + + // F3: main publishes PutPredictedAsync as Predicted and records the prediction whatever instance it is given. The + // shape rule refuses a door whose instance is already the cached one, so a mutate-then-predict is published as + // Updated, is logged on a primary, and its reconciliation is published as Updated. + [Test, Category("SoulFinding")] + public async Task F3_APredictionOfTheCachedInstanceIsPublishedAsAPrediction() + { + var cache = new CultCache(); + var db = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = db.Shards[0]; + var kinds = new List(); + db.Watch().Subscribe(c => kinds.Add(c.Kind)); + await db.PutPredictedAsync(One, Note("p1")); + var held = cache.Get(One)!; + held.Text = "p2"; + await db.PutPredictedAsync(One, held); + var auth = db.Documents.CreateRawDocumentPutMessage("a", new CultRecordHandle(One), Note("auth")); + auth.ShardId = shard.ShardId; + auth.ShardEpoch = shard.Epoch; + await db.ApplyPutAsync(auth); + TestContext.Out.WriteLine($"kinds=[{string.Join(",", kinds)}] log=[{string.Join(",", db.GetMutationLog(shard.ShardId).Select(e => e.Kind))}]"); + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(db.GetMutationLog(shard.ShardId).Select(e => e.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + // F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an + // unrelated key waits for the whole append. + [Test, Category("SoulFinding"), Ignore("Accepted follow-up F4: a log append runs under the cache gate, so readers wait; documented on ICultNetShardMutationLogStore.")] + public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend() + { + var store = new Store { AppendDelayMs = 400 }; + var cache = new CultCache(); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + var db = Db(cache, true, store); + var writer = Task.Run(() => cache.UpsertAsync(Note("one"), new CultRecordHandle(One)).GetAwaiter().GetResult()); + Assert.That(store.InAppend.Wait(5000), Is.True); + var clock = Stopwatch.StartNew(); + _ = cache.Get(Two); + var waited = clock.Elapsed.TotalMilliseconds; + await writer; + TestContext.Out.WriteLine($"reader of an unrelated key waited {waited:0.0} ms for a 400 ms append"); + Assert.That(waited, Is.LessThan(100)); + } + + // P1: the constructor registers the observer before the journal, so a change stashed by the journal always finds + // its publisher, even while another thread writes the cache during construction. + [Test, Category("SoulPin")] + public void P1_ConstructionUnderConcurrentWritesOrphansNoStash() + { + var orphans = 0; + for (var round = 0; round < 200; round++) + { + var cache = new CultCache(); + using var stop = new CancellationTokenSource(); + var writer = Task.Run(() => + { + var i = 0; + while (!stop.IsCancellationRequested) + cache.UpsertAsync(Note("w" + i++), new CultRecordHandle(new CultRecordKey("soul:p4:race:" + (i % 4)))).GetAwaiter().GetResult(); + }); + Thread.SpinWait(200 * (round % 7)); + var db = Db(cache, true, null); + Thread.SpinWait(2000); + stop.Cancel(); + writer.GetAwaiter().GetResult(); + orphans += StashCount(db); + db.Dispose(); + } + + Assert.That(orphans, Is.EqualTo(0), "no stash entry outlives its admission"); + } + + // P2: every stash entry is removed by its publication. + [Test, Category("SoulPin")] + public async Task P2_TheStashDrainsAfterEveryWrite() + { + var cache = new CultCache(); + var db = Db(cache, true, new Store()); + await db.PutAsync(One, Note("one")); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + cache.Commit(batch => batch.Upsert(Note("three"), new CultRecordHandle(Three))); + await db.DeleteAsync(One); + Assert.That(StashCount(db), Is.EqualTo(0)); + } + + // P3: a replica never applies an entry past a hole. The primary burns 2 with compaction refused, then logs 3; a + // replica at 1 is answered compacted_history and resyncs, never applying 3 on top of 1. + [Test, Category("SoulPin")] + public async Task P3_AReplicaAtTheHoleResyncsRatherThanSkippingIt() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + using var peer = new Peer(cache, primary); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + await primary.PutAsync(Three, Note("three")); + Assert.That(peer.Log(1).Reason, Is.EqualTo("compacted_history")); + var applied = await replicator.PullOnceAsync(ShardId); + Assert.That(applied, Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion; writes a log file.")] + public async Task M1_BareWriteLatencyWithTheFileLogStore() + { + var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N")); + try + { + var cache = new CultCache(); + _ = Db(cache, true, new CultNetFileShardMutationLogStore(root)); + var times = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await cache.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + times.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"file store bare write ms: first100 avg {times.Take(100).Average():0.000}, 400-500 avg {times.Skip(400).Take(100).Average():0.000}, last100 avg {times.Skip(900).Average():0.000}, max {times.Max():0.0}"); + var bare = new CultCache(); + var t2 = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await bare.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + t2.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"no database bare write ms: last100 avg {t2.Skip(900).Average():0.000}"); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, true); + } + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs new file mode 100644 index 00000000..a4796e7d --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs @@ -0,0 +1,142 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4c probes. They guard: a committed put whose shard log failed is not answered as a refusal (Q1); reads and watch agree + // on what a database serves (Q2); a replica log apply leaves out a row its shard does not own (Q3); the unowned refusal is not a + // shard-authority exception (Q4); two databases over one cache aggregate their log failures (Q5). + public sealed class SoulP4cProbeTests + { + private const string ShardId = "soulc-shard"; + private static readonly CultRecordKey One = new("soulc:one"); + private static readonly CultRecordKey Unowned = new("soulc:unowned"); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private sealed class RefusingStore : ICultNetShardMutationLogStore + { + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) => Array.Empty(); + public void Append(string shardId, CultNetShardLogEntryMessage entry) => throw new IOException("refused " + entry.Sequence); + public long GetCompactedThrough(string shardId) => 0; + public void CompactThrough(string shardId, long sequence) { } + } + + private sealed class Logger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + // Q1: the server's own remote-put path is a writer. A put that committed and could not be logged is answered as a + // failure: the peer is told its write failed, while the cache holds it and it was published. + [Test, Category("SoulFinding")] + public async Task Q1_ServerAnswersACommittedButUnloggedRemotePutAsAFailure() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, new RefusingStore()); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new Logger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + + try { await handler(message, peer); } catch (Exception) { } + + TestContext.WriteLine("errors: " + string.Join(" | ", logger.Errors)); + Assert.That(cache.Get(One)?.Text, Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Has.None.Contains("raw put failed"), "a committed put must not be reported to the peer as a failed one"); + } + + // Q2: reads serve a cached row no shard owns, the database watch never publishes its changes: a view built from + // GetAll + Watch (CultMesh Collection, the subscription server's snapshot + WatchAllChanges) goes stale. + [Test, Category("SoulFinding")] + public async Task Q2_ReadsServeAnUnownedRowWhoseChangesTheWatchNeverPublishes() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var seen = new List(); + database.Watch().Subscribe(change => seen.Add(change.Document?.Body ?? "")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v1" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v2" }, new CultRecordHandle(Unowned)); + + var read = database.GetAll().Select(note => note.Body).ToArray(); + TestContext.WriteLine($"GetAll={string.Join(",", read)} watch={string.Join(",", seen)}"); + Assert.That(read.Length == 0 || seen.Count > 0, Is.True, "a row the database reads out must be one whose changes it publishes"); + } + + // Q3: a replica's log apply door writes a row its shard does not own. The snapshot apply path filters by + // ShardMatchesRawDocument; the log path does not. + [Test, Category("SoulFinding")] + public async Task Q3_AReplicaLogApplyWritesARowItsShardDoesNotOwn() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, store: null); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "r", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + Exception? thrown = null; + try + { + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = DateTimeOffset.UtcNow.ToString("O"), ChangeKind = "added", Put = put }] + }); + } + catch (Exception exception) { thrown = exception; } + + TestContext.WriteLine($"thrown={thrown?.GetType().Name}: {thrown?.Message}; cached={cache.Get(Unowned)?.Body}; log={replica.GetMutationLog(ShardId).Count}"); + Assert.That(cache.Get(Unowned), Is.Null, "no shard owns it here, so no path writes it"); + } + + // Q4 (pin): the unowned refusal is not a shard-authority exception, so the server's routing path does not see it. + [Test, Category("SoulPin")] + public void Q4_TheUnownedRefusalIsNotAShardAuthorityException() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var put = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "w" }); + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(put))!; + Assert.That(refusal, Is.Not.InstanceOf()); + } + + // Q5 (pin): two databases over one cache, both refusing: a writer's catch (CultNetShardLogException) misses. + [Test, Category("SoulPin")] + public void Q5_TwoDatabasesOnOneCacheAggregateTheirLogFailures() + { + var cache = new CultCache(); + _ = Db(cache, primary: true, new RefusingStore()); + _ = Db(cache, primary: true, new RefusingStore()); + var failure = Assert.Throws(() => cache.Commit(batch => + batch.Upsert(Note("x"), new CultRecordHandle(One))))!; + Assert.That(failure.InnerExceptions.OfType().Count(), Is.EqualTo(2)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs new file mode 100644 index 00000000..6623f8a1 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs @@ -0,0 +1,201 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4d probes. They guard the replica log relay (a skipped entry is relayed as the primary sent it, in memory and on + // disk), key-prefixed shard reads, and measure two costs; the measurements are explicit. + public sealed class SoulP4dProbeTests + { + private const string ShardId = "soupd-shard"; + private static readonly string Now = DateTimeOffset.UtcNow.ToString("O"); + + private static string NoteSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + private static string MeshSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, string[] schemas, string? keyPrefix = null, ICultNetShardMutationLogStore? store = null) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: schemas, keyPrefix: keyPrefix)], + MutationLogStore = store + }); + + private static CultNetShardLogResponseMessage UnownedPutLog(CultNetDatabase anyDb, string key) + { + var put = anyDb.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey(key)), new MeshQuickstartNote { NoteId = "p", Body = "primary-put" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + return new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = Now, ChangeKind = "added", Put = put }] + }; + } + + // Q3a: a replica (no store) that skips an unowned put relays it to a chained replica as a delete. + [Test, Category("SoulFinding")] + public async Task Q3a_MemoryLogRelaysASkippedPutAsARemoval() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)]); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + TestContext.Out.WriteLine($"memory relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null} delete={relayed[0].Delete != null}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"), "the relayed entry keeps the primary's kind"); + } + + // Q3b: the same replica with a durable store relays the primary's put (disk and memory disagree). + [Test, Category("SoulFinding")] + public async Task Q3b_StoreBackedLogRelaysTheSkippedPutAsAPut() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-" + Guid.NewGuid().ToString("N")); + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)], store: new CultNetFileShardMutationLogStore(dir)); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + var memory = replica.GetMutationLog(ShardId); + TestContext.Out.WriteLine($"store relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null}; memory entry kind={memory[0].Kind} doc={(memory[0].Document == null ? "null" : "set")}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added")); + } + + // Q3c: a chained replica that does own the row (config drift) and holds it deletes it on the relayed entry. + [Test, Category("SoulFinding")] + public async Task Q3c_AChainedReplicaThatOwnsTheRowDeletesWhatThePrimaryPut() + { + var cacheA = new CultCache(); + var a = Db(cacheA, primary: false, [NoteSchema(cacheA)]); + await a.ApplyShardLogResponseAsync(UnownedPutLog(a, "relay:k")); + + var cacheC = new CultCache(); + var c = Db(cacheC, primary: false, [NoteSchema(cacheC), MeshSchema(cacheC)]); + await cacheC.UpsertAsync(new MeshQuickstartNote { NoteId = "p", Body = "earlier" }, new CultRecordHandle(new CultRecordKey("relay:k"))); + await c.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "chain", + ShardId = ShardId, + ShardEpoch = 1, + Entries = a.GetMutationLogMessages(ShardId).ToArray() + }); + var row = cacheC.Get(new CultRecordKey("relay:k")); + TestContext.Out.WriteLine($"chained row after relay: {row?.Body ?? ""}"); + Assert.That(row, Is.Not.Null, "the primary put this row; a relay must not delete it"); + } + + // A skipped entry is relayed exactly as the primary sent it, whether the replica keeps its log in memory or on disk. + [Test] + public async Task SkippedEntriesAreRelayedIdenticallyFromMemoryAndFromDisk() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-parity-" + Guid.NewGuid().ToString("N")); + try + { + var memoryCache = new CultCache(); + var inMemory = Db(memoryCache, primary: false, [NoteSchema(memoryCache)]); + var diskCache = new CultCache(); + var onDisk = Db(diskCache, primary: false, [NoteSchema(diskCache)], store: new CultNetFileShardMutationLogStore(dir)); + var log = UnownedPutLog(inMemory, "relay:k"); + var removal = new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = Now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = memoryCache.Registry.GetRequired().SchemaId, + RecordKey = "relay:k", + ShardId = ShardId, + ShardEpoch = 1 + } + }; + log.Entries = [log.Entries[0], removal]; + + await inMemory.ApplyShardLogResponseAsync(log); + await onDisk.ApplyShardLogResponseAsync(log); + + var memory = inMemory.GetMutationLogMessages(ShardId); + var disk = onDisk.GetMutationLogMessages(ShardId); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(disk.Select(e => e.ChangeKind))); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(new[] { true, false })); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(disk.Select(e => e.Put != null))); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(new[] { false, true })); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(disk.Select(e => e.Delete != null))); + Assert.That(memory[0].Put!.Document!.RecordKey, Is.EqualTo(disk[0].Put!.Document!.RecordKey)); + Assert.That(memory[0].Put!.Document!.Payload, Is.EqualTo(disk[0].Put!.Document!.Payload), "the document bytes are the primary's"); + } + finally + { + if (Directory.Exists(dir)) Directory.Delete(dir, true); + } + } + + // Q2: key-prefixed shard: the schema is owned, a key outside the prefix is not. + [Test, Category("SoulPin")] + public async Task Q2_KeyPrefixedReadsServeOnlyTheOwnedKeys() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, [NoteSchema(cache)], keyPrefix: "a:"); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "a" }, new CultRecordHandle(new CultRecordKey("a:1"))); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "b" }, new CultRecordHandle(new CultRecordKey("b:1"))); + Assert.That(database.GetAll().Select(n => n.Text), Is.EqualTo(new[] { "a" })); + Assert.That(await database.GetAsync(new CultRecordKey("b:1")), Is.Null); + Assert.That(await database.GetAsync(new CultRecordKey("a:1")), Is.Not.Null); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var snapshot = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + Assert.That(snapshot.Documents.Select(d => d.RecordKey), Is.EqualTo(new[] { "a:1" })); + } + + [Test, Category("SoulPin")] + public async Task Q2_PrimaryAllServesEveryCachedRow() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { Shards = [CultNetShardDescriptor.PrimaryAll()] }); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "x", Body = "x" }, new CultRecordHandle(new CultRecordKey("any:1"))); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(database.GetByName("x"), Is.Not.Null); + } + + // Q2 cost: GetAll through the database versus the cache, scoped shard (reflection per row). + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion.")] + public void Q2_GetAllCost() + { + var cache = new CultCache(); + var database = Db(cache, primary: false, [NoteSchema(cache)], keyPrefix: "n:"); + cache.Commit(batch => + { + for (var i = 0; i < 20000; i++) + batch.Upsert(new NetworkSchemaNote { Text = "t" + i }, new CultRecordHandle(new CultRecordKey("n:" + i))); + }); + _ = cache.GetAll().Count(); + _ = database.GetAll().Count(); + var sw = Stopwatch.StartNew(); + for (var r = 0; r < 5; r++) _ = cache.GetAll().Count(); + var cacheMs = sw.Elapsed.TotalMilliseconds / 5; + sw.Restart(); + for (var r = 0; r < 5; r++) _ = database.GetAll().Count(); + var dbMs = sw.Elapsed.TotalMilliseconds / 5; + TestContext.Out.WriteLine($"GetAll 20000 rows: cache={cacheMs:F2}ms database={dbMs:F2}ms ratio={dbMs / Math.Max(cacheMs, 0.001):F1}x"); + } + + // Wire: the bytes a peer receives for the unowned refusal. + [Test, Category("SoulMeasure"), Explicit("Prints the wire bytes the Rust captured-bytes case pins.")] + public void Wire_UnownedSchemaBytes() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + var bytes = MessagePack.MessagePackSerializer.Serialize(message, CultNetSchemaMessageSerialization.Options); + TestContext.Out.WriteLine("HEX=" + Convert.ToHexString(bytes)); + TestContext.Out.WriteLine("JSON=" + MessagePack.MessagePackSerializer.ConvertToJson(bytes)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulPub3Tests.cs b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs new file mode 100644 index 00000000..dacfd470 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs @@ -0,0 +1,276 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + [CultDocument("tests.soul_throwing_name", "tests.soul_throwing_name.v1")] + [MessagePack.MessagePackObject] + public sealed class SoulThrowingName + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultName] + public string Name + { + get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text; + set { } + } + } + + [NonParallelizable] + public sealed class SoulPub3Tests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId => CultDocumentRegistry.Shared.GetRequired().SchemaId; + + // Another thread becomes the drainer and parks inside a subscriber on the change "slow". + private static (Task Writer, ManualResetEventSlim Release) BlockDrainer(CultNetDatabase database, CultCache cache, out IDisposable subscription) + { + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + subscription = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "slow") + { + entered.Set(); + release.Wait(Wait); + } + }); + var writer = Task.Run(() => cache.UpsertAsync(Note("slow"), new CultRecordHandle(new CultRecordKey("soul3:slow"))).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True, "drainer parked"); + return (writer, release); + } + + // P1: a write that returns has been logged and published. + [Test] + public async Task P1_AWriteThatReturnedIsLoggedAndPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue<(string? Text, int Thread)>(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue((change.Document?.Text, Environment.CurrentManagedThreadId))); + var (writer, release) = BlockDrainer(database, cache, out var block); + var fast = new CultRecordKey("soul3:fast"); + var writerThread = 0; + await Task.Run(async () => { writerThread = Environment.CurrentManagedThreadId; await database.PutAsync(fast, Note("fast")); }); + + var logged = database.LastWriteSequence(SchemaId, fast); + var published = seen.Any(entry => entry.Text == "fast"); + release.Set(); + await writer; + block.Dispose(); + var deliveredOn = seen.First(entry => entry.Text == "fast").Thread; + TestContext.Progress.WriteLine($"SOUL P1 logged-at-return={logged} published-at-return={published} writer={writerThread} deliveredOn={deliveredOn}"); + Assert.Multiple(() => + { + Assert.That(logged, Is.Not.Null, "PutAsync returned before its change was logged"); + Assert.That(published, Is.True, "PutAsync returned before its change was published"); + }); + } + + // P2: a subscriber that waits (bounded) for another thread's write to be observed. + [Test] + public async Task P2_ASubscriberWaitingForAnotherThreadsWriteIsNotStarved() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var other = new ManualResetEventSlim(); + var observedInTime = false; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "other") + other.Set(); + if (change.Document?.Text == "trigger") + { + Task.Run(() => database.PutAsync(new CultRecordKey("soul3:other"), Note("other"))); + observedInTime = other.Wait(TimeSpan.FromSeconds(2)); + } + }); + await database.PutAsync(new CultRecordKey("soul3:trigger"), Note("trigger")); + Assert.That(observedInTime, Is.True, "another thread's committed write never reached subscribers while this handler ran"); + } + + // P3: one observer's escaped exception (fail-fast R3 handler) loses one change; it must not stall all later ones. + [Test] + public async Task P3_AnEscapedObserverExceptionDoesNotStallLaterChanges() + { + var previous = ObservableSystem.GetUnhandledExceptionHandler(); + ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); + try + { + var cache = new CultCache(); + using var thrower = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom"); + }); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try { await cache.UpsertAsync(Note("boom"), new CultRecordHandle(new CultRecordKey("soul3:boom"))); } + catch (InvalidOperationException) { } + + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + var after = new CultRecordKey("soul3:after"); + await database.PutAsync(after, Note("after")); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + finally + { + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + } + } + + // P4: an admission that throws inside Apply after minting a Sequence (a [CultName] getter) must not stall later changes. + [Test] + public async Task P4_AThrowMidApplyDoesNotStallLaterChanges() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try + { + cache.Commit(batch => + { + batch.Upsert(new SoulThrowingName { Text = "ok" }, new CultRecordHandle(new CultRecordKey("soul3:ok"))); + batch.Upsert(new SoulThrowingName { Text = "boom" }, new CultRecordHandle(new CultRecordKey("soul3:boomname"))); + }); + } + catch (InvalidOperationException) { } + + var after = new CultRecordKey("soul3:after-apply"); + await database.PutAsync(after, Note("after")); + TestContext.Progress.WriteLine($"SOUL P4 ok-in-cache={cache.Get(new CultRecordKey("soul3:ok")) != null}"); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + + // P5: writers no longer carry their own delivery; the backlog behind one drainer grows without bound. + [Test] + public async Task P5_OtherWritersDoNotPileAnUnboundedBacklogOnOneDrainer() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var delivered = 0; + var backlogAtBDone = -1; + const int bWrites = 2000; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text?.StartsWith("b") == true) Interlocked.Increment(ref delivered); + if (change.Document?.Text == "a") + { + Task.Run(async () => + { + for (var i = 0; i < bWrites; i++) + await database.PutAsync(new CultRecordKey($"soul3:b:{i}"), Note($"b{i}")); + }).Wait(Wait); + backlogAtBDone = bWrites - Volatile.Read(ref delivered); + } + }); + await database.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + TestContext.Progress.WriteLine($"SOUL P5 undelivered after B's {bWrites} writes returned: {backlogAtBDone}"); + Assert.That(backlogAtBDone, Is.EqualTo(0)); + } + + // P6: a prediction still queued when the authoritative put arrives. + [Test] + public async Task P6_AnAuthoritativePutArrivingBeforeItsPredictionIsDeliveredStillReconciles() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul3:predict"); + var kinds = new ConcurrentQueue(); + using var watch = database.WatchRecord(key).Subscribe(change => kinds.Enqueue(change.Kind)); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + var (writer, release) = BlockDrainer(database, cache, out var block); + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + release.Set(); + await writer; + block.Dispose(); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + // P7: a replica's absent-key delete is logged ahead of an earlier entry still queued; a bounded pull then sees a gap. + [Test] + public async Task P7_AReplicaLogStaysInSequenceOrderWhenAnAbsentDeleteOvertakesAQueuedPut() + { + const string shardId = "chain3"; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [SchemaId])] + }); + var p = Make(new CultCache(), true); + var rCache = new CultCache(); + var r = Make(rCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to, int? limit = null) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId), limit).ToArray() + }); + } + + var z = new CultRecordKey("soul3:z"); + await p.PutAsync(z, Note("z")); + await Pull(p, r); + await Pull(r, c); + rCache.Remove(z); + await p.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + await p.DeleteAsync(z); + + var (writer, release) = BlockDrainer(r, rCache, out var block); + await Pull(p, r); + release.Set(); + await writer; + block.Dispose(); + + TestContext.Progress.WriteLine($"SOUL P7 r log order: {string.Join(",", r.GetMutationLog(shardId).Select(e => e.Sequence))}"); + Assert.Multiple(() => + { + Assert.That(r.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.DoesNotThrowAsync(() => Pull(r, c, limit: 1), "a bounded pull from R"); + }); + } + } +}