diff --git a/CultLib.sln b/CultLib.sln index 217f2d1f..a8baa383 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -51,6 +51,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.ConsumerBu EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.GeneratorConsumer", "tests\GameCult.Caching.GeneratorConsumer\GameCult.Caching.GeneratorConsumer.csproj", "{1A75EA8F-00B3-49AB-8398-B1B7723BE10B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.OutsideStoreTests", "tests\GameCult.Caching.OutsideStoreTests\GameCult.Caching.OutsideStoreTests.csproj", "{A54C2C29-941A-46A9-9DDE-9711B2F2EADD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -289,6 +291,18 @@ Global {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x64.Build.0 = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.ActiveCfg = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -314,5 +328,6 @@ Global {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} {D3138F3C-9531-4906-A5D3-74BD857D0685} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {1A75EA8F-00B3-49AB-8398-B1B7723BE10B} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} EndGlobalSection EndGlobal diff --git a/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj new file mode 100644 index 00000000..c40c048d --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj @@ -0,0 +1,22 @@ + + + + + net10.0 + false + enable + GameCult.Caching.OutsideStoreTests + + + + + + + + + + + + + + diff --git a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs new file mode 100644 index 00000000..150fdf4d --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs @@ -0,0 +1,359 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.OutsideStoreTests +{ + // variants:ruling:store-kind-per-key-judge: a store kind written outside the library applies the whole per-key ownership rule + // through the one protected judge (CacheBackingStore.RefuseUndeclaredOnDisk), using only public and protected members. This + // assembly has no InternalsVisibleTo, so a judge parameter type, or something it needs, turning internal breaks it here. + public class OutsideStoreJudgeTests + { + [CultDocument("tests.out_a", "tests.out_a.v1")] + [MessagePackObject] + public sealed class OutA + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.out_b", "tests.out_b.v1")] + [MessagePackObject] + public sealed class OutB + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // RbX's schema id, which RbY lists; the test below asserts it, so a changed derivation fails there. + private const string RbXId = "sha256:25087e0d76fd5b9ac950723ccd47f3f3ac1428b53b49ca9bda9aae8b38a0220a"; + + [CultDocument("tests.rb_x", "tests.rb_x.v1")] + [MessagePackObject] + public sealed class RbX + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { RbXId })] + [MessagePackObject] + public sealed class RbY + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_z", "tests.rb_z.v1")] + [MessagePackObject] + public sealed class RbZ + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // What the "disk" holds: records by key and the catalog that publishes their ids. + private sealed class Disk + { + public readonly Dictionary Records = new(); + public readonly List Catalog = new(); + private int _version; + + public void Put(CultPersistedRecord record, CultSchemaCatalogEntry entry) + { + if (Catalog.All(known => known.SchemaId != entry.SchemaId)) + Catalog.Add(entry); + Records[record.Key] = (record, (++_version).ToString()); + } + + public string? SchemaIdAt(string key) => Records.TryGetValue(key, out var held) ? held.Record.SchemaId : null; + } + + // A store kind with no InternalsVisibleTo: an in-memory disk, flushed and committed under the protected judge. + private sealed class OutsideStore : CacheBackingStore + { + private readonly Disk _disk; + private readonly CultCodec _codec; + private readonly HashSet _staged = new(StringComparer.Ordinal); + + public OutsideStore(Disk disk, CultCodec codec) + { + _disk = disk; + _codec = codec; + } + + private object Decode(Type type, byte[] payload) => _codec.Deserialize(type, type, payload)!; + + private CultStoredDocument? DurableAt(string key) + { + if (!_disk.Records.TryGetValue(key, out var held)) + return null; + try + { + return ToStoredDocument(held.Record, held.Version, _disk.Catalog, Decode, out _); + } + catch (Exception) + { + return null; + } + } + + public override void PullAll() => Held(() => + { + var loaded = _disk.Records + .Where(pair => !Entries.TryGetValue(pair.Key, out var known) || !SameStoredIdentity(known.StoredVersion, pair.Value.Version)) + .Select(pair => DurableAt(pair.Key)) + .OfType() + .ToArray(); + var dropped = Entries.Values.Where(known => !_disk.Records.ContainsKey(known.Key.Value)).ToArray(); + if (loaded.Length > 0 || dropped.Length > 0) + Loaded?.Invoke(loaded, dropped); + foreach (var stored in dropped) + Entries.TryRemove(stored.Key.Value, out _); + foreach (var stored in loaded) + Entries[stored.Key.Value] = stored; + }); + + public override void Push(CultStoredDocument entry) => Held(() => + { + Entries[entry.Key.Value] = entry; + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void Delete(CultStoredDocument entry) => Held(() => + { + Entries.TryRemove(entry.Key.Value, out _); + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void PushAll() => Held(() => Apply(Array.Empty(), Array.Empty(), Array.Empty())); + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() => + { + Apply(request.Upserts, request.Deletes, request.Checks); + foreach (var entry in request.Deletes) + Entries.TryRemove(entry.Key.Value, out _); + foreach (var entry in request.Upserts) + Entries[entry.Key.Value] = entry; + return CultCommitOutcome.Committed; + }); + + private void Apply(IReadOnlyList upserts, IReadOnlyList deletes, IEnumerable checks) + { + var writes = upserts.ToDictionary(entry => entry.Key.Value, StringComparer.Ordinal); + foreach (var key in _staged) + { + if (Entries.TryGetValue(key, out var pushed)) + writes.TryAdd(key, pushed); + } + + var removals = deletes.Select(entry => entry.Key.Value) + .Concat(_staged.Where(key => !writes.ContainsKey(key) && !Entries.ContainsKey(key))) + .ToArray(); + var judged = removals.Concat(checks).ToArray(); + var entries = writes.ToDictionary(pair => pair.Key, pair => pair.Value.Descriptor.ToCatalogEntry(), StringComparer.Ordinal); + + // The whole rule, from the protected judge: the same call both shipped stores make. + RefuseUndeclaredOnDisk( + writes.Keys.Concat(judged).Concat(StagedChecks), + _disk.Catalog, + key => _disk.Records.TryGetValue(key, out var held) ? held.Record : null, + key => entries.TryGetValue(key, out var entry) ? entry : null, + judged, + DurableAt, + _ => null, + key => + { + _staged.Remove(key); + IsDirty = _staged.Count > 0; + }); + + foreach (var (key, document) in writes) + { + var type = document.Descriptor.DocumentType; + _disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = document.Descriptor.SchemaId, + StoredAt = document.StoredAt, + Payload = _codec.Serialize(document.Document, type, type) + }, + entries[key]); + } + + foreach (var key in removals.Where(key => !writes.ContainsKey(key))) + _disk.Records.Remove(key); + _staged.Clear(); + MarkFlushSucceeded(); + } + } + + private static readonly CultDocumentRegistry AB = CultDocumentRegistry.ForTypes(new[] { typeof(OutA), typeof(OutB) }); + private static readonly CultDocumentRegistry Rb = CultDocumentRegistry.ForTypes(new[] { typeof(RbX), typeof(RbY), typeof(RbZ) }); + private static readonly CultRecordKey K = new("k"); + private static readonly CultRecordKey K2 = new("k2"); + + private static CultCache Open(Disk disk, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(new OutsideStore(disk, cache.Codec!)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + // Another writer puts a record of this type at the key on the disk, as the cache's own store would have. + private static void PutOnDisk(Disk disk, CultDocumentRegistry registry, string key, T document, string? asSchemaId = null, CultSchemaCatalogEntry? entry = null) + where T : class + { + var codec = CultCacheMessagePack.CreateCodec(registry); + var descriptor = registry.GetRequired(); + disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = asSchemaId ?? descriptor.SchemaId, + StoredAt = DateTimeOffset.UtcNow.ToString("O"), + Payload = codec.Serialize(document, typeof(T), typeof(T)) + }, + entry ?? descriptor.ToCatalogEntry()); + } + + private static Type HeldType(CultCache cache, CultRecordKey key) => cache.GetStored(key)!.Descriptor.DocumentType; + + // The written type's own id: a record of another registered type that this cache never loaded. + [Test] + public void AnotherRegisteredTypesUnheldRecordIsRefusedUnderTheLockAndReloaded() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + var before = disk.Records["k"]; + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), + Throws.InstanceOf()); + Assert.That(disk.Records["k"], Is.EqualTo(before), "the refused write changed nothing on the disk"); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB)), "the refusal reloaded the record the store holds"); + } + + [Test] + public void AFlushOfAnotherRegisteredTypesUnheldRecordIsRefusedToo() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + cache.UpsertAsync(typeof(OutA), new OutA { Name = "k", Value = 1 }, K).GetAwaiter().GetResult(); + Assert.That(() => cache.FlushAsync().GetAwaiter().GetResult(), Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + } + + // Claimant precedence, a listed id: the written type owns a record under an id it lists, and a type that lists nothing does not. + [Test] + public void ARecordUnderAnIdTheWrittenTypeListsIsOwnedAndOneItDoesNotListIsRefused() + { + Assert.That(Rb.GetRequired().SchemaId, Is.EqualTo(RbXId)); + var disk = new Disk(); + using var cache = Open(disk, Rb); + PutOnDisk(disk, Rb, "k", new RbX { Name = "k" }); + PutOnDisk(disk, Rb, "k2", new RbX { Name = "k2" }); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(RbY), new RbY { Name = "k", Value = 1 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(Rb.GetRequired().SchemaId)); + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(RbZ), new RbZ { Name = "k2" }, K2)), + Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k2"), Is.EqualTo(RbXId), "the type that lists nothing replaced nothing"); + } + + // Reads whole: an older id of the written type's own schema name, which the type reads whole, is its own and is restamped. + [Test] + public void AnOlderIdOfTheWrittenSchemaNameThatIsReadWholeIsOwnedAndRestamped() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var current = AB.GetRequired().ToCatalogEntry(); + var older = new CultSchemaCatalogEntry + { + SchemaId = "sha256:older-out-a", + SchemaName = current.SchemaName, + SchemaVersion = "tests.out_a.v0", + ContentHash = current.ContentHash, + CanonicalSchemaJson = current.CanonicalSchemaJson, + Members = current.Members + }; + PutOnDisk(disk, AB, "k", new OutA { Name = "k" }, asSchemaId: older.SchemaId, entry: older); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 3 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(current.SchemaId)); + } + + // Staged retype removals: a held key retypes as a removal then an add, in one batch and staged apart. + [TestCase(true)] + [TestCase(false)] + public void ARetypeOfAHeldKeyCommitsInOneBatchAndStagedApart(bool oneBatch) + { + var disk = new Disk(); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + using var cache = Open(disk, AB); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True); + else + { + Assert.That(cache.Remove(K), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), Is.True); + } + + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutA))); + } + + [Test] + public void ARetypeOfAKeyTheCacheNeverLoadedIsJudgedByTheRemovalRule() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True, "a record a registered type reads may be removed, so the key retypes"); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + } + + // A removal is never licensed over a record no registered type owns, retype or not. + [Test] + public void ACommitNeverRemovesOrRetypesARecordNoRegisteredTypeOwns() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var unknown = new CultSchemaCatalogEntry { SchemaId = "tests.out_unknown", SchemaName = "tests.out_unknown" }; + disk.Put(new CultPersistedRecord { Key = "k", SchemaId = unknown.SchemaId, StoredAt = "t", Payload = new byte[] { 0x90 } }, unknown); + var before = disk.Records["k"]; + + Assert.That(() => cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k" }, K); + }), Throws.InstanceOf(), "retype"); + Assert.That(() => cache.Commit(batch => batch.Remove(K)), Throws.InstanceOf(), "removal"); + Assert.That(disk.Records["k"], Is.EqualTo(before)); + } + } +}