diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 167218d9..dba918ba 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -178,7 +178,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore if (commitLease == null) return CultCommitOutcome.Contended; var manifest = ReadManifest(); - if (!request.ConditionsHold(manifest.Records, Entries.Values)) + if (!request.ConditionsHold(manifest.Records, Entries.Values, Registry.CanonicalSchemaId)) return CultCommitOutcome.Mismatch; var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 79bc50e8..dc1de3a6 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -129,6 +129,13 @@ record and when it puts one. The cache is the only owner of that stamp: a store changes the identity of a record it did not receive from its caller, so a record another writer changed since this cache read it is written back as that writer left it. +A schema type declares the older ids its records may sit under: `CultDocumentAttribute.CompatibleSchemaIds` +(C#), `compatibleSchemaIds` (TypeScript, Python). The registered catalog entry lists them after its own id, and +the cache restamps a record loaded under one to the type's own id, so the next write of that record carries the +registered id. A record another writer left under a declared id, and that a commit does not rewrite, stays under it +and the registered entry publishes it. Without the declaration, a C# commit onto a file that holds a record under an +id only the file's entry lists is refused, typed, and the file is left as it was. Rust has no such declaration. + A cache resolves a record to a local type by the record's schema id: the registered type that owns the id, else one that lists it as compatible. The name the catalog entry carries is metadata. It names a local type only when no local type has the id, for a store written by a runtime whose @@ -151,9 +158,8 @@ an own id and disagree on the schema name, when records of different types share write would change which type an existing record resolves to (the schema names of the records already in the store and of the records written differ), or when a record's id is published by no chosen entry. The retyping refusal has one exception, a rename: when the entries of both the existing records and -the written ones own the id, the registered descriptor wins. (A C# cache stamps a record with its -descriptor's own id, so it cannot write a record under an id its descriptor only lists, and the -retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the +the written ones own the id, the registered descriptor wins. (A C# cache stamps every record it +writes with its descriptor's own id, so the retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError` (Rust, TypeScript, Python). diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index a1db32f9..202d869a 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -241,6 +241,7 @@ namespace GameCult.Caching string schemaName, string schemaVersion, string schemaId, + IReadOnlyList compatibleSchemaIds, string contentHash, string canonicalSchemaJson, bool isGlobal, @@ -254,6 +255,7 @@ namespace GameCult.Caching SchemaName = schemaName; SchemaVersion = schemaVersion; SchemaId = schemaId; + CompatibleSchemaIds = compatibleSchemaIds; ContentHash = contentHash; CanonicalSchemaJson = canonicalSchemaJson; IsGlobal = isGlobal; @@ -279,6 +281,9 @@ namespace GameCult.Caching public string SchemaName { get; } public string SchemaVersion { get; } public string SchemaId { get; } + + /// The ids besides this type declared through . + public IReadOnlyList CompatibleSchemaIds { get; } public string ContentHash { get; } public string CanonicalSchemaJson { get; } public bool IsGlobal { get; } @@ -370,7 +375,7 @@ namespace GameCult.Caching SchemaVersion = SchemaVersion, ContentHash = ContentHash, CanonicalSchemaJson = CanonicalSchemaJson, - CompatibleSchemaIds = [SchemaId], + CompatibleSchemaIds = [SchemaId, .. CompatibleSchemaIds], Members = Members .OrderBy(member => member.Slot) .Select(member => new CultSchemaMemberCatalogEntry @@ -578,6 +583,16 @@ namespace GameCult.Caching throw new InvalidOperationException($"Unknown CultCache schema id '{schemaId}'."); } + // The id a record under schemaId is stamped with: the own id of the type that owns it, else of the first registered + // type that declares it compatible; an id no registered type claims is its own. + internal string CanonicalSchemaId(string schemaId) + { + var indexes = _indexes; + return indexes.BySchemaId.TryGetValue(schemaId, out var owner) || indexes.ByCompatibleSchemaId.TryGetValue(schemaId, out owner) + ? owner.SchemaId + : schemaId; + } + public CultDocumentDescriptor ResolvePersistedSchema(string schemaId, IReadOnlyCollection catalog) { return ResolvePersistedSchemaDetailed(schemaId, catalog).Descriptor; @@ -613,6 +628,11 @@ namespace GameCult.Caching throw new InvalidOperationException($"Persisted schema '{schemaId}' is not present in the embedded catalog."); } + if (indexes.ByCompatibleSchemaId.TryGetValue(schemaId, out var declared)) + { + return BuildCompatibleResolutionResult(persisted, declared); + } + foreach (var compatibleSchemaId in persisted.CompatibleSchemaIds.Where(candidate => !string.IsNullOrWhiteSpace(candidate))) { if (indexes.BySchemaId.TryGetValue(compatibleSchemaId, out var compatibleLocal)) @@ -687,6 +707,8 @@ namespace GameCult.Caching indexes.ByType[descriptor.DocumentType] = descriptor; indexes.BySchemaId[descriptor.SchemaId] = descriptor; + foreach (var compatibleSchemaId in descriptor.CompatibleSchemaIds) + indexes.ByCompatibleSchemaId.TryAdd(compatibleSchemaId, descriptor); indexes.BySchemaName[descriptor.SchemaName] = schemaNameVersions == null ? [descriptor] : [.. schemaNameVersions, descriptor]; @@ -717,6 +739,7 @@ namespace GameCult.Caching { ByType = new Dictionary(); BySchemaId = new Dictionary(StringComparer.Ordinal); + ByCompatibleSchemaId = new Dictionary(StringComparer.Ordinal); BySchemaName = new Dictionary(StringComparer.Ordinal); } @@ -724,11 +747,13 @@ namespace GameCult.Caching { ByType = new Dictionary(source.ByType); BySchemaId = new Dictionary(source.BySchemaId, StringComparer.Ordinal); + ByCompatibleSchemaId = new Dictionary(source.ByCompatibleSchemaId, StringComparer.Ordinal); BySchemaName = new Dictionary(source.BySchemaName, StringComparer.Ordinal); } public Dictionary ByType { get; } public Dictionary BySchemaId { get; } + public Dictionary ByCompatibleSchemaId { get; } public Dictionary BySchemaName { get; } } @@ -765,12 +790,19 @@ namespace GameCult.Caching var contentHash = Sha256(schemaJson); var semanticFingerprint = BuildSemanticFingerprint(attribute.SchemaName, attribute.SchemaVersion, descriptorMembers); var schemaId = Sha256(semanticFingerprint); + if (attribute.CompatibleSchemaIds.Any(string.IsNullOrWhiteSpace)) + throw new InvalidOperationException($"Type {type.FullName} declares an empty compatible schema id."); + var compatibleSchemaIds = attribute.CompatibleSchemaIds + .Where(id => !string.Equals(id, schemaId, StringComparison.Ordinal)) + .Distinct(StringComparer.Ordinal) + .ToArray(); return new CultDocumentDescriptor( type, attribute.SchemaName, attribute.SchemaVersion, schemaId, + compatibleSchemaIds, contentHash, schemaJson, type.GetCustomAttribute() != null, @@ -1783,7 +1815,7 @@ namespace GameCult.Caching // durable is what the store holds on disk now; observed is what its cache last loaded or committed. // Identity is (schemaId, storedAt), sound because every write to a key mints a later storedAt. - public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed) + public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed, Func canonicalSchemaId) { var byKey = durable.ToDictionary(record => record.Key, StringComparer.Ordinal); foreach (var (key, schemaId, storedAt) in Expected) @@ -1791,13 +1823,13 @@ namespace GameCult.Caching byKey.TryGetValue(key.Value, out var record); var holds = schemaId == null ? record == null - : record != null && record.SchemaId == schemaId && record.StoredAt == storedAt; + : record != null && canonicalSchemaId(record.SchemaId) == schemaId && record.StoredAt == storedAt; if (!holds) return false; } return !ExpectUnchanged || - durable.Select(record => Identity(record.Key, record.SchemaId, record.StoredAt)) + durable.Select(record => Identity(record.Key, canonicalSchemaId(record.SchemaId), record.StoredAt)) .OrderBy(identity => identity, StringComparer.Ordinal) .SequenceEqual(observed .Select(stored => Identity(stored.Key.Value, stored.Descriptor.SchemaId, stored.StoredAt)) @@ -2542,7 +2574,7 @@ namespace GameCult.Caching var inMemory = _entries.Values .Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt }) .ToArray(); - return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + return request.ConditionsHold(inMemory, _entries.Values, Registry.CanonicalSchemaId) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; }); }); } @@ -3607,7 +3639,7 @@ namespace GameCult.Caching return CultCommitOutcome.Contended; var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); - if (!request.ConditionsHold(disk.Records, Entries.Values)) + if (!request.ConditionsHold(disk.Records, Entries.Values, Registry.CanonicalSchemaId)) return CultCommitOutcome.Mismatch; // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, @@ -3664,7 +3696,7 @@ namespace GameCult.Caching { if (landing.Contains(record.Key) || removing.Contains(record.Key)) continue; - if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == record.SchemaId) + if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == Registry.CanonicalSchemaId(record.SchemaId)) continue; arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _)); } diff --git a/src/GameCult.Caching/CultDocumentContracts.cs b/src/GameCult.Caching/CultDocumentContracts.cs index 442f5c4f..b439ac9f 100644 --- a/src/GameCult.Caching/CultDocumentContracts.cs +++ b/src/GameCult.Caching/CultDocumentContracts.cs @@ -18,6 +18,14 @@ namespace GameCult.Caching public string SchemaName { get; } public string SchemaVersion { get; } + + /// + /// Schema ids other than this type's own that a store may hold its records under, typically the ids an earlier + /// version of the schema had. The registered catalog entry lists them, and a cache resolves a record stored under + /// one of them to this type (its own id is tried first) and restamps it to the type's own id on load. TypeScript + /// and Python name the same declaration compatibleSchemaIds. + /// + public string[] CompatibleSchemaIds { get; set; } = Array.Empty(); } [AttributeUsage(AttributeTargets.Class, Inherited = false)] diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs new file mode 100644 index 00000000..c82b10bd --- /dev/null +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -0,0 +1,272 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // A document type declares the older schema ids its records may sit under (CultDocumentAttribute.CompatibleSchemaIds). The + // registered catalog entry lists them, a cache resolves a record under one to that type and stamps it with the type's own id, + // and a store writer never changes the identity of a record it did not receive. + public class CompatibleSchemaIdTests + { + private const string OldId = "old.id"; + private const string OwnerId = "sha256:86186abf7d63d8a326c725d39a10f3a4db11b1e8698e8c0fcf6d9720031bc480"; + private const string SharedId = "shared.id"; + + [CultDocument("tests.compat_deck", "tests.compat_deck.v2", CompatibleSchemaIds = new[] { OldId })] + [MessagePackObject] + public sealed class DeclaringDeck + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + // The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing. + [CultDocument("tests.compat_deck", "tests.compat_deck.v2")] + [MessagePackObject] + public sealed class BareDeck + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.compat_owner", "tests.compat_owner.v1", CompatibleSchemaIds = new[] { OwnerId })] + [MessagePackObject] + public sealed class Owner + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.compat_claimant", "tests.compat_claimant.v1", CompatibleSchemaIds = new[] { OwnerId, SharedId })] + [MessagePackObject] + public sealed class Claimant + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.compat_second_claimant", "tests.compat_second_claimant.v1", CompatibleSchemaIds = new[] { SharedId })] + [MessagePackObject] + public sealed class SecondClaimant + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.compat_empty", "tests.compat_empty.v1", CompatibleSchemaIds = new[] { " " })] + [MessagePackObject] + public sealed class EmptyClaim + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + private static readonly CultDocumentRegistry Declaring = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaringDeck) }); + private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { typeof(BareDeck) }); + private static readonly CultRecordKey D = new("d"); + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-compat-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private static CultCache Open(string path, CultDocumentRegistry registry) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // A store written by a runtime that knew the schema under an older id and another name: one entry that owns OldId, one + // record under it. Nothing but the declaration lets this runtime's type claim it. + private string OldStore(string name) + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); + var snapshot = Read(path); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OldId; + entry.SchemaName = "tests.legacy_deck"; + entry.CompatibleSchemaIds = new[] { OldId }; + snapshot.Records.Single().SchemaId = OldId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // The same store with the entry keeping this schema's name and only listing the older id: what a store carries after a + // writer that did not declare it, so a runtime without the declaration can still read it. + private string OldStoreUnderTheSameName(string name) + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); + var snapshot = Read(path); + var entry = snapshot.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; + snapshot.Records.Single().SchemaId = OldId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + [Test] + public void ARecordUnderADeclaredOldIdOpensAndACommitRewritesItUnderTheRegisteredId() + { + var path = OldStore("old.cc"); + var descriptor = Declaring.GetRequired(); + + using (var cache = Open(path, Declaring)) + { + Assert.That(cache.Get(D)!.Name, Is.EqualTo("d"), "the declaration resolves the record to the type"); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d2" }, D)), Is.True); + } + + var written = Read(path); + Assert.That(written.Records.Single().SchemaId, Is.EqualTo(descriptor.SchemaId)); + var entry = written.SchemaCatalog.Single(); + Assert.That(entry.SchemaId, Is.EqualTo(descriptor.SchemaId)); + Assert.That(entry.SchemaName, Is.EqualTo("tests.compat_deck")); + Assert.That(entry.CompatibleSchemaIds, Is.EqualTo(new[] { descriptor.SchemaId, OldId })); + foreach (var registry in new[] { Declaring, Bare }) + { + using var reopened = Open(path, registry); + Assert.That(reopened.AllEntries.Count(), Is.EqualTo(1)); + } + } + + [Test] + public void ARecordUnderAnOldIdNoTypeDeclaresIsNotOpenedByThatSchemaName() + { + var path = OldStore("undeclared.cc"); + + Assert.That(() => Open(path, Bare), Throws.InstanceOf(), "the entry's name is metadata; only a declared id resolves it"); + } + + // An unconditional commit writes the cache's whole view: the record the batch does not name is stamped with the registered + // id too, because the cache stamped it when it loaded it. + [Test] + public void ACommitThatWritesTheWholeViewStampsEveryLoadedRecordWithTheRegisteredId() + { + var path = OldStore("whole-view.cc"); + var descriptor = Declaring.GetRequired(); + + using (var cache = Open(path, Declaring)) + cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); + + var written = Read(path); + Assert.That(written.Records.Select(record => record.SchemaId), Is.All.EqualTo(descriptor.SchemaId)); + Assert.That(written.Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e" })); + } + + // A commit onto the file writes its batch and leaves every other record as the file holds it: the store writer never + // changes identity. The record stays under the older id, which the registered entry publishes as compatible. + [Test] + public void ACommitOntoTheFileLeavesARecordUnderADeclaredOldIdAsItIsAndPublishesTheId() + { + var path = OldStoreUnderTheSameName("onto.cc"); + var descriptor = Declaring.GetRequired(); + + using (var cache = Open(path, Declaring)) + { + Assert.That(cache.Get(D), Is.Not.Null); + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.True); + } + + var written = Read(path); + Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId)); + Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId)); + Assert.That(written.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain(OldId)); + using var reopened = Open(path, Declaring); + Assert.That(reopened.Get(D), Is.Not.Null); + } + + [Test] + public void WithoutTheDeclarationACommitOntoTheFileIsRefusedTypedAndTheFileLeftAsItWas() + { + var path = OldStoreUnderTheSameName("undeclared-onto.cc"); + var bytes = File.ReadAllBytes(path); + + using var cache = Open(path, Bare); + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(BareDeck), new BareDeck { Name = "e" }, new CultRecordKey("e")); + }))!; + Assert.That(refusal.SchemaId, Is.EqualTo(OldId)); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + } + + // A record under a declared old id is the same record the cache holds under the registered id: a condition naming it holds. + [Test] + public void AConditionOnARecordUnderADeclaredOldIdHoldsAndTheWholeStoreCountsAsUnchanged() + { + var path = OldStoreUnderTheSameName("conditions.cc"); + + using var cache = Open(path, Declaring); + var current = cache.Get(D)!; + Assert.That(cache.TryCommit(batch => + { + batch.Expect(D, current); + batch.ExpectUnchanged(); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.EqualTo(CultCommitOutcome.Committed)); + } + + [Test] + public void ADeclarationIsRegisteredWithoutItsOwnIdAndItsEntryListsTheOwnIdFirst() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Owner) }); + var descriptor = registry.GetRequired(); + + Assert.That(descriptor.SchemaId, Is.EqualTo(OwnerId), "the pinned id is this schema's id"); + Assert.That(descriptor.CompatibleSchemaIds, Is.Empty, "a type's own id is not a declared compatible id"); + Assert.That(descriptor.ToCatalogEntry().CompatibleSchemaIds, Is.EqualTo(new[] { OwnerId })); + } + + [Test] + public void AnIdATypeOwnsResolvesToItEvenWhenAnEarlierTypeDeclaresItCompatible() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(Owner) }); + var owner = registry.GetRequired(); + var entry = owner.ToCatalogEntry(); + + Assert.That(registry.CanonicalSchemaId(OwnerId), Is.EqualTo(OwnerId)); + Assert.That(registry.ResolvePersistedSchema(OwnerId, new[] { entry }), Is.SameAs(owner)); + } + + [Test] + public void AnIdTwoTypesDeclareCompatibleResolvesToTheOneRegisteredFirst() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(SecondClaimant) }); + var first = registry.GetRequired(); + + Assert.That(registry.CanonicalSchemaId(SharedId), Is.EqualTo(first.SchemaId)); + } + + [Test] + public void AnIdNoTypeClaimsIsItsOwnCanonicalId() + { + Assert.That(Declaring.CanonicalSchemaId("nobody.id"), Is.EqualTo("nobody.id")); + } + + [Test] + public void AnEmptyDeclaredIdIsRefused() + { + Assert.That(() => CultDocumentRegistry.ForTypes(new[] { typeof(EmptyClaim) }).GetRequired(), + Throws.InvalidOperationException.With.Message.Contains("empty compatible schema id")); + } + } +}