diff --git a/docs/start/README.md b/docs/start/README.md index 9ca7f499..e9dc0b3f 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -16,7 +16,7 @@ this page. | | CultCache | CultMath | Geometry | CultNet | CultMesh | | --- | --- | --- | --- | --- | --- | -| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | +| **C#** | [first call, tested](#c-first-call); [update, tested](#c-update-with-retry) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | | **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [first call, tested](#typescript-cultnet-first-call); [package](../../packages/cultnet-ts/README.md) | [package](../../packages/cultmesh-ts/README.md); snippet not yet; [recipe: two-client .io game](io-game.md), tested with two live clients, browser page not yet run | | **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [first call, tested](#rust-cultnet-first-call); [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples) | [package](../../packages/cultmesh-rs); snippet not yet | | **Python** | [first call, tested](#python-first-call) | not yet | not yet | [first call, tested](#python-cultnet-first-call); [package](../../packages/cultnet-py/README.md) | [package](../../packages/cultmesh-py/README.md); snippet not yet | @@ -70,6 +70,37 @@ public static class FirstCall `FirstCall.Run` is the call; a second test in that file fails if the block above stops matching it. More: `src/GameCult.Caching/README.md`. +## C# update with retry + +Change a record you may not be the only one changing. `Update` reads the record, hands your function a private +copy, and commits the result only if the record is still the one it read. If another writer got there first, it +reads again and runs your function again, up to five times. The source is +[`tests/GameCult.Caching.Tests/UpdateCallTests.cs`](../../tests/GameCult.Caching.Tests/UpdateCallTests.cs); the +snippet reuses `Item` from the first call. + +```csharp +using GameCult.Caching; +using GameCult.Caching.MessagePack; + +public static class UpdateCall +{ + public static CultUpdateResult Restock(string path, CultRecordKey key) + { + var cache = CultCacheMessagePack.Create(path); + return cache.Update(key, item => + { + item ??= new Item { Name = "Potion", Category = "Consumable" }; + item.Value += 10; + return item; + }); + } +} +``` + +The write is durable when `Update` returns, with no `PushAll`. A lost race re-runs the function, so keep it free of +side effects; when the attempts run out the result says `LostRace` instead of throwing. `UpsertAsync` plus +`PushAll` is the blind opt-out: last writer wins. + ## TypeScript first call Same shape in TypeScript: the document is a name, a schema id and a `parse`-able schema (zod here), diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 469fc7af..dc0c8b99 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2052,6 +2052,29 @@ namespace GameCult.Caching Contended } + public enum CultUpdateOutcome + { + Updated, + Declined, + LostRace + } + + /// + /// What did. A lost race is a value, not an exception. is the instance the + /// cache holds at the key when Update returns: the written one for Updated, null when the key is absent. + /// + public readonly struct CultUpdateResult where T : class + { + public CultUpdateResult(CultUpdateOutcome outcome, T? document) + { + Outcome = outcome; + Document = document; + } + + public CultUpdateOutcome Outcome { get; } + public T? Document { get; } + } + public sealed class CultCommitRequest { internal CultCommitRequest( @@ -2563,6 +2586,7 @@ namespace GameCult.Caching return UpsertAsync(document, handle); } + // A blind write, last-writer-wins per key; Update is the conditional golden path. // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares. public Task> UpsertAsync(T document, CultRecordHandle? handle = null) { @@ -2639,6 +2663,7 @@ namespace GameCult.Caching /// /// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published. + /// Without Expect a commit is blind; retries a conditional one. /// /// /// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer @@ -2656,6 +2681,59 @@ namespace GameCult.Caching return Land(stage, wait: false); } + /// + /// Reads the record at , applies to a private copy of it, and commits the result only + /// if the record is still the one that was read; when another writer got there first it re-reads and runs + /// again, up to times. The write is durable when Update returns. + /// + /// + /// may run up to times. It gets a private copy (null when the key holds + /// nothing), which it may change and return. It must not write to the cache or have other side effects. Return null to write + /// nothing. A store with staged blind writes refuses Update until it is flushed, as it refuses any conditional commit. The cache + /// needs a codec to copy with. Running out of attempts returns ; every other failure throws. + /// + public CultUpdateResult Update(CultRecordKey key, Func apply, int maxAttempts = 5) where T : class + { + if (apply == null) throw new ArgumentNullException(nameof(apply)); + if (maxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required."); + var codec = _codec ?? throw new InvalidOperationException("This cache has no codec; create it with CultCacheMessagePack.Create."); + for (var attempt = 1; ; attempt++) + { + var (version, copy) = Held(() => + { + if (!_entries.TryGetValue(key.Value, out var held)) + return ((string?)null, (T?)null); + if (held.Variant != null || held.Document is not T) + throw new InvalidOperationException($"Record {key.Value} is a variant or not a {typeof(T).Name}; Update cannot copy it."); + return (held.StoredVersion, (T?)codec.Clone(held.Document, held.Descriptor.DocumentType)); + }); + var next = apply(copy); + if (next == null) + return new CultUpdateResult(CultUpdateOutcome.Declined, Get(key)); + CultCommitOutcome outcome; + try + { + outcome = Land(batch => + { + batch.Expected.Add((key, version)); + batch.Upsert(next, new CultRecordHandle(key)); + }, wait: true); + } + catch (CultWriteConflictException conflict) when (conflict.InnerException == null) + { + outcome = CultCommitOutcome.Mismatch; + } + + if (outcome == CultCommitOutcome.Committed) + return new CultUpdateResult(CultUpdateOutcome.Updated, next); + if (outcome != CultCommitOutcome.Mismatch) + throw new InvalidOperationException($"Update of {key.Value} was refused: {outcome}."); + Held(() => { Home(next.GetType())?.PullAll(); return true; }); + if (attempt >= maxAttempts) + return new CultUpdateResult(CultUpdateOutcome.LostRace, Get(key)); + } + } + public CultRecordHandle? TryGetHandle(T document) { if (document == null) throw new ArgumentNullException(nameof(document)); diff --git a/src/GameCult.Caching/CultInspectorModel.cs b/src/GameCult.Caching/CultInspectorModel.cs index 7aa6ab74..60dec974 100644 --- a/src/GameCult.Caching/CultInspectorModel.cs +++ b/src/GameCult.Caching/CultInspectorModel.cs @@ -352,7 +352,7 @@ namespace GameCult.Caching } } - public object Clone(object document, Type type) => _deserialize(type, _serialize(document, type, type)); + public object Clone(object document, Type type) => _codec.Clone(document, type); public CultInspectorEdit BeginEdit(CultStoredDocument record) { diff --git a/src/GameCult.Caching/CultVariants.cs b/src/GameCult.Caching/CultVariants.cs index 48909c19..e5c28bbf 100644 --- a/src/GameCult.Caching/CultVariants.cs +++ b/src/GameCult.Caching/CultVariants.cs @@ -29,6 +29,11 @@ namespace GameCult.Caching // (document payload, slot and encoded value pairs) public Func>, byte[]> Overlay { get; } + // The one copy of a document: its encoding, decoded again. Update's private argument and CultInspectorModel.Clone share it. + internal object Clone(object document, Type type) => + Deserialize(type, type, Serialize(document, type, type)) + ?? throw new InvalidOperationException($"The codec returned null for {type.FullName}."); + /// /// Decodes a document payload read from a store file, as a cache load decodes it: the ids a load mints are minted from the key /// and their path, so every reader of one store mints the same ids, and a payload a cache write would refuse is refused here too. diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index 6c76f5cd..dffce80a 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -13,6 +13,7 @@ Cacheable models are plain classes annotated with cache intent. using GameCult.Caching; using MessagePack; +[MessagePackObject] [CultDocument("gamecult.item_data", "gamecult.item_data.v1")] public sealed class ItemData { @@ -131,18 +132,33 @@ using GameCult.Caching; using GameCult.Caching.MessagePack; var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack"); +var key = new CultRecordKey("potion"); +// The golden path: read, change a private copy, commit only if the record is unchanged, re-run on a lost race. +// Durable when it returns; no flush. +var result = cache.Update(key, item => +{ + item ??= new ItemData { Name = "Potion", Sku = "POT-1", Category = "Consumable" }; + item.Value += 10; + return item; +}); + +// The blind opt-out: last writer wins per key. Name the key, or a new key claims the same unique Sku and is refused. var handle = await cache.UpsertAsync(new ItemData { Name = "Potion", Sku = "POT-1", Category = "Consumable", Value = 50 -}); +}, new CultRecordHandle(key)); await cache.FlushAsync(); ``` +`Update` may run your function more than once, so it must have no side effects. Returning null writes nothing, and a +lost race that outlasts `maxAttempts` comes back as `CultUpdateOutcome.LostRace`. `UpsertAsync` plus `FlushAsync` never +checks what it overwrites. + If you want the explicit assembly rite, the lower-level `CultCache` + `SingleFileMessagePackBackingStore` path is still there. The helper is just the blessed front door. @@ -294,13 +310,10 @@ Useful surfaces: - `cache.IsDirty` - `store.IsDirty` - `cache.Document(key)` -- `cache.Soa()` -- `cache.UpsertAsync(document)` +- `cache.Update(key, apply)` +- `cache.UpsertAsync(document)` (blind write) - `cache.FlushAllBackingStores()` - `cache.FlushAsync()` -- `cache.FlushBackingStore(store)` -- `cache.PrepareForReloadOrShutdown()` -- `cache.PrepareForReloadOrShutdownAsync()` - `cache.FlushAttachedStoresOnDispose` - `store.FlushOnDispose` diff --git a/tests/GameCult.Caching.Tests/FirstCallTests.cs b/tests/GameCult.Caching.Tests/FirstCallTests.cs index 8fad1229..d8febc49 100644 --- a/tests/GameCult.Caching.Tests/FirstCallTests.cs +++ b/tests/GameCult.Caching.Tests/FirstCallTests.cs @@ -62,17 +62,28 @@ namespace GameCult.Caching.Tests } [Test] - public void DocsShowTheTestedSnippet() - { - var source = File.ReadAllText(SourcePath()); - const string begin = "// snippet:begin\n"; - var start = source.IndexOf(begin, StringComparison.Ordinal) + begin.Length; - var snippet = source[start..source.IndexOf("// snippet:end\n", StringComparison.Ordinal)]; - var docs = File.ReadAllText(Path.Combine(Path.GetDirectoryName(SourcePath())!, "..", "..", "docs", "start", "README.md")); - Assert.That(docs.Replace("\r\n", "\n"), Does.Contain("```csharp\n" + snippet + "```"), - "docs/start/README.md no longer shows tests/GameCult.Caching.Tests/FirstCallTests.cs verbatim"); - } + public void DocsShowTheTestedSnippet() => StartDocs.AssertShowsSnippet(SourcePath()); static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; } + + // docs/start/README.md shows each tested snippet verbatim: the region between snippet:begin and snippet:end of its source file. + internal static class StartDocs + { + public static void AssertShowsSnippet(string sourcePath) => + AssertShowsSnippet(sourcePath, Path.Combine("..", "..", "docs", "start", "README.md")); + + // The same check against another document, its path relative to the directory of sourcePath. The snippet there is the + // statements of a block, so it need not open the fence (the block's usings come first). + public static void AssertShowsSnippet(string sourcePath, string docsRelativePath, bool wholeBlock = true) + { + var source = File.ReadAllText(sourcePath).Replace("\r\n", "\n"); + const string begin = "// snippet:begin\n"; + var start = source.IndexOf(begin, StringComparison.Ordinal) + begin.Length; + var snippet = source[start..source.IndexOf("// snippet:end\n", StringComparison.Ordinal)]; + var docs = File.ReadAllText(Path.Combine(Path.GetDirectoryName(sourcePath)!, docsRelativePath)); + Assert.That(docs.Replace("\r\n", "\n"), Does.Contain(wholeBlock ? "```csharp\n" + snippet + "```" : snippet), + $"{docsRelativePath} no longer shows {Path.GetFileName(sourcePath)} verbatim"); + } + } } diff --git a/tests/GameCult.Caching.Tests/ReadmeHappyPathTests.cs b/tests/GameCult.Caching.Tests/ReadmeHappyPathTests.cs new file mode 100644 index 00000000..de2dabb4 --- /dev/null +++ b/tests/GameCult.Caching.Tests/ReadmeHappyPathTests.cs @@ -0,0 +1,81 @@ +#nullable enable +using System; +using System.IO; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The Happy Path of src/GameCult.Caching/README.md, run as written: its statements are the region between snippet:begin and + // snippet:end below, and ReadmeShowsTheTestedSnippet fails when the README stops showing them verbatim. + public class ReadmeHappyPathTests + { + [CultDocument("gamecult.item_data", "gamecult.item_data.v1")] + [MessagePackObject] + public sealed class ItemData + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] [CultIndex] public string Sku = string.Empty; + [Key(2)] public string Category = string.Empty; + [Key(3)] public int Value; + } + + private static async Task HappyPath() + { +// snippet:begin +var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack"); +var key = new CultRecordKey("potion"); + +// The golden path: read, change a private copy, commit only if the record is unchanged, re-run on a lost race. +// Durable when it returns; no flush. +var result = cache.Update(key, item => +{ + item ??= new ItemData { Name = "Potion", Sku = "POT-1", Category = "Consumable" }; + item.Value += 10; + return item; +}); + +// The blind opt-out: last writer wins per key. Name the key, or a new key claims the same unique Sku and is refused. +var handle = await cache.UpsertAsync(new ItemData +{ + Name = "Potion", + Sku = "POT-1", + Category = "Consumable", + Value = 50 +}, new CultRecordHandle(key)); + +await cache.FlushAsync(); +// snippet:end + return cache; + } + + [Test] + public async Task HappyPathRunsAsWritten() + { + var directory = Path.Combine(Path.GetTempPath(), $"readme-happy-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + var previous = Environment.CurrentDirectory; + try + { + Environment.CurrentDirectory = directory; // the README opens a relative path + using var cache = await HappyPath(); + Assert.That(cache.Get(new CultRecordKey("potion"))!.Value, Is.EqualTo(50)); + Assert.That(cache.GetByIndex("Sku", "POT-1"), Is.Not.Null); + } + finally + { + Environment.CurrentDirectory = previous; + Directory.Delete(directory, recursive: true); + } + } + + [Test] + public void ReadmeShowsTheTestedSnippet() => + StartDocs.AssertShowsSnippet(SourcePath(), Path.Combine("..", "..", "src", "GameCult.Caching", "README.md"), wholeBlock: false); + + static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; + } +} diff --git a/tests/GameCult.Caching.Tests/UpdateCallTests.cs b/tests/GameCult.Caching.Tests/UpdateCallTests.cs new file mode 100644 index 00000000..d9a8b31e --- /dev/null +++ b/tests/GameCult.Caching.Tests/UpdateCallTests.cs @@ -0,0 +1,54 @@ +#nullable enable +using System; +using System.IO; +using NUnit.Framework; + +// snippet:begin +using GameCult.Caching; +using GameCult.Caching.MessagePack; + +public static class UpdateCall +{ + public static CultUpdateResult Restock(string path, CultRecordKey key) + { + var cache = CultCacheMessagePack.Create(path); + return cache.Update(key, item => + { + item ??= new Item { Name = "Potion", Category = "Consumable" }; + item.Value += 10; + return item; + }); + } +} +// snippet:end + +namespace GameCult.Caching.Tests +{ + // The C# update snippet that docs/start/README.md shows verbatim; it reuses FirstCallTests' Item. + public class UpdateCallTests + { + [Test] + public void RestockTwiceReadsTwenty() + { + var path = Path.Combine(Path.GetTempPath(), $"update-call-{Guid.NewGuid():N}.cc"); + try + { + var key = new CultRecordKey("potion"); + Assert.That(UpdateCall.Restock(path, key).Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(UpdateCall.Restock(path, key).Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + + using var reader = CultCacheMessagePack.Create(path); + Assert.That(reader.Get(key)!.Value, Is.EqualTo(20)); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void DocsShowTheTestedSnippet() => StartDocs.AssertShowsSnippet(SourcePath()); + + static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; + } +} diff --git a/tests/GameCult.Caching.Tests/UpdateTests.cs b/tests/GameCult.Caching.Tests/UpdateTests.cs new file mode 100644 index 00000000..0dd96bad --- /dev/null +++ b/tests/GameCult.Caching.Tests/UpdateTests.cs @@ -0,0 +1,515 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // CultCache.Update: read the content version, apply a pure function to a private copy, commit conditionally, re-run on a lost + // race (variants:ruling:write-set-content-version-declared-ownership; variants:question:update-apply-copy). + public class UpdateTests + { + private static readonly CultRecordKey Key = new("counter"); + private static readonly CultDocumentRegistry Registry = + CultDocumentRegistry.ForTypes(new[] { typeof(UpdateCounter), typeof(UpdateTally), typeof(UpdateGear) }); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-update-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, bool directory, bool codec = true) + { + var cache = codec ? new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)) : new CultCache(Registry); + cache.AddBackingStore(directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path)); + return cache; + } + + private static void Seed(string path, bool directory, int value) + { + using var cache = Open(path, directory); + Assert.That(cache.Commit(batch => batch.Upsert(new UpdateCounter { Name = "counter", Value = value }, new CultRecordHandle(Key))), Is.True); + } + + private static void Rival(CultCache cache, int value) => + Assert.That(cache.Commit(batch => batch.Upsert(new UpdateCounter { Name = "counter", Value = value }, new CultRecordHandle(Key))), Is.True); + + private static int StoredValue(string path, bool directory) + { + using var reader = Open(path, directory); + return reader.Get(Key)!.Value; + } + + private static UpdateCounter Plus(UpdateCounter? counter, int by) + { + counter ??= new UpdateCounter { Name = "counter" }; + counter.Value += by; + return counter; + } + + [TestCase(false)] + [TestCase(true)] + public void TwoWritersUpdatingLoseNoIncrement(bool directory) + { + const int n = 100; + var path = PathOf("race.cc"); + Seed(path, directory, 0); + using var a = Open(path, directory); + using var b = Open(path, directory); + var applied = 0; + + var racers = new[] { a, b }.Select(cache => Task.Run(() => + { + var results = new List(); + for (var i = 0; i < n; i++) + results.Add(cache.Update(Key, counter => + { + Interlocked.Increment(ref applied); + return Plus(counter, 1); + }, maxAttempts: 200).Outcome); + return results; + })).ToArray(); + Assert.That(Task.WaitAll(racers, TimeSpan.FromMinutes(2)), Is.True, "the racers did not finish"); + + Assert.That(racers.SelectMany(racer => racer.Result), Is.All.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(StoredValue(path, directory), Is.EqualTo(2 * n)); + Assert.That(applied, Is.GreaterThan(2 * n), "the writers never actually raced"); + } + + [Test] + public void LostAttemptRerunsApplyOnTheFreshValue() + { + var path = PathOf("rerun.cc"); + Seed(path, directory: false, 1); + using var cache = Open(path, false); + using var rival = Open(path, false); + var seen = new List(); + + var result = cache.Update(Key, counter => + { + seen.Add(counter!.Value); + if (seen.Count == 1) + Rival(rival, 100); + return Plus(counter, 1); + }); + + Assert.That(seen, Is.EqualTo(new[] { 1, 100 })); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(result.Document!.Value, Is.EqualTo(101)); + Assert.That(StoredValue(path, false), Is.EqualTo(101)); + } + + [Test] + public void ExhaustedAttemptsReturnLostRace() + { + var path = PathOf("lost.cc"); + Seed(path, false, 1); + using var cache = Open(path, false); + using var rival = Open(path, false); + var calls = 0; + + var result = cache.Update(Key, counter => + { + Rival(rival, 100 * ++calls); + return Plus(counter, 1); + }, maxAttempts: 3); + + Assert.That(calls, Is.EqualTo(3)); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.LostRace)); + Assert.That(result.Document!.Value, Is.EqualTo(300)); + Assert.That(StoredValue(path, false), Is.EqualTo(300), "the store holds the rival's last write"); + } + + [Test] + public void ApplyGetsAPrivateCopy() + { + var path = PathOf("copy.cc"); + Seed(path, false, 1); + using var cache = Open(path, false); + using var rival = Open(path, false); + var published = new List(); + using var subscription = cache.Watch().Subscribe(change => published.Add(change.Document!.Value)); + var arguments = new List(); + var calls = 0; + + var result = cache.Update(Key, counter => + { + calls++; + arguments.Add(counter!.Value); + Assert.That(counter, Is.Not.SameAs(cache.Get(Key)), "apply was handed the held instance"); + counter.Value += 1000; + Assert.That(cache.Get(Key)!.Value, Is.Not.EqualTo(counter.Value), "the change reached the cache before any commit"); + if (calls == 1) + Rival(rival, 5); + return counter; + }); + + Assert.That(arguments, Is.EqualTo(new[] { 1, 5 }), "the second call saw the first call's change"); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(published, Does.Not.Contain(1001), "an uncommitted value was published"); + Assert.That(published.Count(value => value == 1005), Is.EqualTo(1)); + Assert.That(StoredValue(path, false), Is.EqualTo(1005)); + } + + [Test] + public void NullFromApplyDeclines() + { + var path = PathOf("decline.cc"); + Seed(path, false, 7); + var before = File.ReadAllBytes(path); + using var cache = Open(path, false); + var changes = 0; + using var subscription = cache.Watch().Subscribe(_ => changes++); + + var result = cache.Update(Key, _ => null); + + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Declined)); + Assert.That(result.Document!.Value, Is.EqualTo(7)); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(changes, Is.EqualTo(0)); + } + + [TestCase(false)] + [TestCase(true)] + public void AbsentKeyCreatesOnceUnderRace(bool directory) + { + var path = PathOf("create.cc"); + using var a = Open(path, directory); + using var b = Open(path, directory); + var started = new Barrier(2); + + var racers = new[] { a, b }.Select(cache => Task.Run(() => + { + started.SignalAndWait(); + return cache.Update(Key, counter => Plus(counter, 1)).Outcome; + })).ToArray(); + Assert.That(Task.WaitAll(racers, TimeSpan.FromMinutes(1)), Is.True); + + Assert.That(racers.Select(racer => racer.Result), Is.All.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(StoredValue(path, directory), Is.EqualTo(2)); + } + + [Test] + public void RefusalsAreNotRetried() + { + var path = PathOf("refuse.cc"); + Seed(path, false, 1); + var before = File.ReadAllBytes(path); + var calls = 0; + Func apply = counter => { calls++; return Plus(counter, 1); }; + + using (var codecLess = Open(path, false, codec: false)) + Assert.That(() => codecLess.Update(Key, apply), Throws.InvalidOperationException); + using (var cache = Open(path, false)) + { + Assert.That(() => cache.Update(Key, apply, maxAttempts: 0), Throws.TypeOf()); + Assert.That(() => cache.Update(Key, null!), Throws.TypeOf()); + Assert.That(() => cache.Update(Key, _ => new UpdateTally { Name = "counter" }), Throws.InvalidOperationException); + } + + using (var cache = Open(path, false)) + { + cache.Commit(batch => + { + batch.Upsert(new UpdateGear { Name = "gear", Power = 1 }, new CultRecordHandle(new CultRecordKey("gear"))); + }); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("gear-big"), new CultRecordKey("gear"), + new[] { cache.Override(nameof(UpdateGear.Name), "gear big"), cache.Override(nameof(UpdateGear.Power), 9) })); + } + + before = File.ReadAllBytes(path); + var gearCalls = 0; + using (var cache = Open(path, false)) + Assert.That(() => cache.Update(new CultRecordKey("gear-big"), gear => { gearCalls++; return gear; }), Throws.InvalidOperationException); + Assert.That(gearCalls, Is.EqualTo(0), "apply ran for a variant"); + Assert.That(calls, Is.EqualTo(0), "apply ran for a refused call"); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + + using (var cache = Open(path, false)) + { + cache.UpsertAsync(new UpdateTally { Name = "staged" }).GetAwaiter().GetResult(); + Assert.That(() => cache.Update(Key, apply), Throws.InvalidOperationException.With.Message.Contains("flush before a conditional commit")); + Assert.That(calls, Is.LessThanOrEqualTo(1)); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "a refusal wrote"); + } + + [Test] + public void VariantStoreConflictIsALostAttempt() + { + var path = PathOf("variant.cc"); + Seed(path, false, 1); + using (var seed = Open(path, false)) + { + seed.Commit(batch => batch.Upsert(new UpdateGear { Name = "gear", Power = 1 }, new CultRecordHandle(new CultRecordKey("gear")))); + seed.Commit(batch => batch.UpsertVariant(new CultRecordKey("gear-big"), new CultRecordKey("gear"), + new[] { seed.Override(nameof(UpdateGear.Name), "gear big"), seed.Override(nameof(UpdateGear.Power), 9) })); + } + + using var cache = Open(path, false); + using (var rival = Open(path, false)) + Assert.That(rival.Commit(batch => batch.Upsert(new UpdateTally { Name = "other" }, new CultRecordHandle(new CultRecordKey("other")))), Is.True); + var calls = 0; + + var result = cache.Update(Key, counter => { calls++; return Plus(counter, 1); }); + + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(calls, Is.EqualTo(2), "the conflict was not a lost attempt that re-ran apply"); + Assert.That(StoredValue(path, false), Is.EqualTo(2)); + } + + private static CultPersistedStoreSnapshot ReadSnapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // A reload that fails inside a conflict is a read failure, not a lost race: Update lets it propagate and never re-runs apply. + // Here the gear a variant is based on has gone foreign in the file, so the reload the conflict owes cannot drop it. + [Test] + public void ConflictWhoseReloadFailedPropagates() + { + var path = PathOf("reload-failed.cc"); + Seed(path, false, 1); + using var cache = Open(path, false); + cache.Commit(batch => batch.Upsert(new UpdateGear { Name = "gear", Power = 1 }, new CultRecordHandle(new CultRecordKey("gear")))); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("gear-big"), new CultRecordKey("gear"), + new[] { cache.Override(nameof(UpdateGear.Name), "gear big"), cache.Override(nameof(UpdateGear.Power), 9) })); + const string foreignId = "sha256:00000000000000000000000000000000000000000000000000000000000050de"; + var snapshot = ReadSnapshot(path); + snapshot.Records.Single(record => record.Key == "gear").SchemaId = foreignId; + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Append(new CultSchemaCatalogEntry + { + SchemaId = foreignId, SchemaName = "tests.update_foreign", SchemaVersion = "tests.update_foreign.v1", + ContentHash = "foreign", CanonicalSchemaJson = "{}" + }).ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var calls = 0; + + var conflict = Assert.Throws(() => cache.Update(Key, counter => { calls++; return Plus(counter, 1); }))!; + + Assert.That(conflict.InnerException, Is.Not.Null, "the reload that failed is the cause"); + Assert.That(calls, Is.EqualTo(1), "a failed reload was retried"); + } + + [TestCase(false)] + [TestCase(true)] + public void RivalWritingAndRestoringTheValueStillRerunsApply(bool directory) + { + var path = PathOf("aba.cc"); + Seed(path, directory, 1); + using var cache = Open(path, directory); + using var rival = Open(path, directory); + var seen = new List(); + + var result = cache.Update(Key, counter => + { + seen.Add(counter!.Value); + if (seen.Count == 1) + { + Rival(rival, 5); + Rival(rival, 1); + } + return Plus(counter, 1); + }); + + Assert.That(seen, Is.EqualTo(new[] { 1, 1 }), "the content version missed A-B-A"); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(StoredValue(path, directory), Is.EqualTo(2)); + } + + [TestCase(false)] + [TestCase(true)] + public void RivalRemovalIsSeenAsNull(bool directory) + { + var path = PathOf("removal.cc"); + Seed(path, directory, 1); + using var cache = Open(path, directory); + using var rival = Open(path, directory); + var seen = new List(); + + var result = cache.Update(Key, counter => + { + seen.Add(counter?.Value); + if (seen.Count == 1) + rival.Commit(batch => batch.Remove(Key)); + return Plus(counter, 1); + }); + + Assert.That(seen, Is.EqualTo(new int?[] { 1, null })); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + Assert.That(StoredValue(path, directory), Is.EqualTo(1)); + } + + [Test] + public void ExhaustedAttemptsReturnLostRaceInADirectoryStore() + { + var path = PathOf("lost-dir.cc"); + Seed(path, true, 1); + using var cache = Open(path, true); + using var rival = Open(path, true); + var calls = 0; + + var result = cache.Update(Key, counter => + { + Rival(rival, 100 * ++calls); + return Plus(counter, 1); + }, maxAttempts: 2); + + Assert.That(calls, Is.EqualTo(2)); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.LostRace)); + Assert.That(result.Document!.Value, Is.EqualTo(200)); + Assert.That(StoredValue(path, true), Is.EqualTo(200)); + } + + [Test] + public void OneAttemptMeansOneApplyAndNonPositiveThrows() + { + var path = PathOf("one.cc"); + Seed(path, false, 1); + using var cache = Open(path, false); + using var rival = Open(path, false); + var calls = 0; + + var result = cache.Update(Key, counter => { calls++; Rival(rival, 50); return Plus(counter, 1); }, maxAttempts: 1); + + Assert.That(calls, Is.EqualTo(1)); + Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.LostRace)); + Assert.That(() => cache.Update(Key, counter => counter, maxAttempts: -1), Throws.TypeOf()); + Assert.That(calls, Is.EqualTo(1)); + } + + [TestCase(false)] + [TestCase(true)] + public void UpdatedDocumentIsTheHeldInstanceAndOwnWritesCauseNoRetry(bool directory) + { + var path = PathOf("own.cc"); + Seed(path, directory, 1); + using var cache = Open(path, directory); + cache.UpsertAsync(new UpdateCounter { Name = "counter", Value = 10 }, new CultRecordHandle(Key)).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + var calls = 0; + + var first = cache.Update(Key, counter => { calls++; return Plus(counter, 1); }); + var second = cache.Update(Key, counter => { calls++; return Plus(counter, 1); }); + + Assert.That(calls, Is.EqualTo(2), "the cache's own writes made a later Update retry"); + Assert.That(first.Document, Is.Not.Null); + Assert.That(second.Document, Is.SameAs(cache.Get(Key)), "Updated returned a copy, not the held instance"); + Assert.That(StoredValue(path, directory), Is.EqualTo(12)); + } + + [Test] + public void ApplyRunsOutsideTheCacheGate() + { + var path = PathOf("gate.cc"); + Seed(path, false, 1); + using var cache = Open(path, false); + var reached = false; + + cache.Update(Key, counter => + { + reached = Task.Run(() => cache.Get(Key)).Wait(TimeSpan.FromSeconds(10)); + return Plus(counter, 1); + }); + + Assert.That(reached, Is.True, "another thread could not read the cache while apply ran"); + } + + [TestCase(false)] + [TestCase(true)] + public void ContendedLedgerStoredValueEqualsUpdatedCount(bool directory) + { + var path = PathOf("ledger.cc"); + Seed(path, directory, 0); + var caches = Enumerable.Range(0, 3).Select(_ => Open(path, directory)).ToArray(); + try + { + var tasks = caches.Select(cache => Task.Run(() => + { + var updated = 0; + for (var i = 0; i < 25; i++) + if (cache.Update(Key, counter => Plus(counter, 1)).Outcome == CultUpdateOutcome.Updated) + updated++; + return updated; + })).ToArray(); + Assert.That(Task.WaitAll(tasks, TimeSpan.FromMinutes(2)), Is.True); + + Assert.That(StoredValue(path, directory), Is.EqualTo(tasks.Sum(task => task.Result)), "an Updated outcome was lost or a LostRace wrote"); + } + finally + { + foreach (var cache in caches) + cache.Dispose(); + } + } + + [Test] + public void InMemoryCacheUpdatesUnderThreadRace() + { + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + cache.Commit(batch => batch.Upsert(new UpdateCounter { Name = "counter" }, new CultRecordHandle(Key))); + + var threads = Enumerable.Range(0, 2).Select(_ => Task.Run(() => + { + for (var i = 0; i < 100; i++) + Assert.That(cache.Update(Key, counter => Plus(counter, 1), maxAttempts: 200).Outcome, Is.EqualTo(CultUpdateOutcome.Updated)); + })).ToArray(); + Assert.That(Task.WaitAll(threads, TimeSpan.FromMinutes(1)), Is.True); + + Assert.That(cache.Get(Key)!.Value, Is.EqualTo(200)); + } + + [CultDocument("tests.update_counter", "tests.update_counter.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class UpdateCounter + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Value; + } + + [CultDocument("tests.update_tally", "tests.update_tally.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class UpdateTally + { + [Key(0)] + [CultName] + public string Name = string.Empty; + } + + [CultDocument("tests.update_gear", "tests.update_gear.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class UpdateGear + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + } + } +}