From 5c8af219f9b68e6edc7f0703d2e736f9f58b985f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:35:40 +0200 Subject: [PATCH] C#: a commit condition compares the id a record was loaded under with the file's, exactly Each stored document carries StoredSchemaId: the id its record sits under in its store, the one it was loaded under until a write puts it under its type's own id. Expect, ExpectUnchanged, the reload and merge identity checks compare (StoredSchemaId, storedAt) with the store's bytes exactly. CanonicalSchemaId is deleted: canonicalising the disk id let an older version's whole-view rewrite (same storedAt, restamped to its own id, a member shed) pass a newer cache's condition (Soul F1), and failed every condition on a record resolved by name or through a file entry listing the local id (F3). A whole-view write marks every record it wrote as stored under its type's id, in the store and the cache. Tests: Soul's rolling-deploy probe as AWholeViewRewriteByAnOlderVersionFails... (Mismatch for Expect and ExpectUnchanged, nothing derived lands, a pull shows the rewrite and the retry commits); F3's H1, H1b, H2 pass Expect and ExpectUnchanged when nothing changed; conditions after a whole-view commit hold for a plain record and a variant. BareDeck is emitted at run time (EmittedDocumentTypes) so the shared registry never sees it beside DeclaringDeck; the CanonicalSchemaId tests and their Claimant fixtures go with the function. Co-Authored-By: Claude Opus 5.5 --- .../DirectoryMessagePackBackingStore.cs | 4 +- src/GameCult.Caching/CultCache.cs | 79 +++-- .../CompatibleSchemaIdTests.cs | 295 +++++++++++++----- .../EmittedDocumentTypes.cs | 56 ++++ 4 files changed, 331 insertions(+), 103 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index dba918ba..d078999a 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -109,7 +109,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore .Where(pair => !Staged(pair.Key) && (!Entries.TryGetValue(pair.Key, out var existing) || existing.StoredAt != pair.Value.StoredAt || - existing.Descriptor.SchemaId != pair.Value.Descriptor.SchemaId)) + existing.StoredSchemaId != pair.Value.StoredSchemaId)) .Select(pair => pair.Value) .ToArray(); var departed = _hydratedKeys @@ -178,7 +178,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore if (commitLease == null) return CultCommitOutcome.Contended; var manifest = ReadManifest(); - if (!request.ConditionsHold(manifest.Records, Entries.Values, Registry.CanonicalSchemaId)) + if (!request.ConditionsHold(manifest.Records, Entries.Values)) return CultCommitOutcome.Mismatch; var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 8d0f6619..48adc00b 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -425,6 +425,7 @@ namespace GameCult.Caching Key = key; StoredAt = storedAt; Descriptor = descriptor; + StoredSchemaId = descriptor.SchemaId; _document = document; } @@ -440,6 +441,7 @@ namespace GameCult.Caching Key = key; StoredAt = storedAt; Descriptor = descriptor; + StoredSchemaId = descriptor.SchemaId; Variant = variant; _document = resolved; _codec = codec; @@ -454,6 +456,11 @@ namespace GameCult.Caching public string StoredAt { get; } public CultDocumentDescriptor Descriptor { get; } + // The schema id the record carries in its store: the id it was loaded under, which may be an older or foreign id that + // resolved to Descriptor, until a write of this record puts it under Descriptor's id. With StoredAt it is the record's + // identity on disk, and a commit condition compares the pair with the store's bytes exactly. + public string StoredSchemaId { get; internal set; } + // The complete document. For a variant, the resolved view: derived, never written back. public object Document => _document ?? throw new InvalidOperationException($"Variant {Key.Value} has not been resolved; only the cache resolves a variant."); @@ -487,7 +494,7 @@ namespace GameCult.Caching } internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly, CultCodec codec) => - new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly }; + new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId }; } public sealed class CultDocumentRegistry @@ -583,16 +590,6 @@ namespace GameCult.Caching throw new InvalidOperationException($"Unknown CultCache schema id '{schemaId}'."); } - // The id a record under schemaId is stamped with: the own id of the type that owns it, else of the first registered - // type that declares it compatible; an id no registered type claims is its own. - internal string CanonicalSchemaId(string schemaId) - { - var indexes = _indexes; - return indexes.BySchemaId.TryGetValue(schemaId, out var owner) || indexes.ByCompatibleSchemaId.TryGetValue(schemaId, out owner) - ? owner.SchemaId - : schemaId; - } - public CultDocumentDescriptor ResolvePersistedSchema(string schemaId, IReadOnlyCollection catalog) { return ResolvePersistedSchemaDetailed(schemaId, catalog).Descriptor; @@ -1807,8 +1804,10 @@ namespace GameCult.Caching public bool HasConditions => Expected.Count > 0 || ExpectUnchanged; // durable is what the store holds on disk now; observed is what its cache last loaded or committed. - // Identity is (schemaId, storedAt), sound because every write to a key mints a later storedAt. - public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed, Func canonicalSchemaId) + // Identity is (stored schema id, storedAt), compared exactly. A write of a key mints a later storedAt, but a whole-view + // write rewrites every record it holds under its own type's id with the storedAt it loaded, and may shed members its + // type lacks: only the id says the bytes changed. + public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed) { var byKey = durable.ToDictionary(record => record.Key, StringComparer.Ordinal); foreach (var (key, schemaId, storedAt) in Expected) @@ -1816,16 +1815,16 @@ namespace GameCult.Caching byKey.TryGetValue(key.Value, out var record); var holds = schemaId == null ? record == null - : record != null && canonicalSchemaId(record.SchemaId) == schemaId && record.StoredAt == storedAt; + : record != null && record.SchemaId == schemaId && record.StoredAt == storedAt; if (!holds) return false; } return !ExpectUnchanged || - durable.Select(record => Identity(record.Key, canonicalSchemaId(record.SchemaId), record.StoredAt)) + durable.Select(record => Identity(record.Key, record.SchemaId, record.StoredAt)) .OrderBy(identity => identity, StringComparer.Ordinal) .SequenceEqual(observed - .Select(stored => Identity(stored.Key.Value, stored.Descriptor.SchemaId, stored.StoredAt)) + .Select(stored => Identity(stored.Key.Value, stored.StoredSchemaId, stored.StoredAt)) .OrderBy(identity => identity, StringComparer.Ordinal)); } @@ -1903,7 +1902,7 @@ namespace GameCult.Caching { ThrowIfSealed(); var observed = _cache.Observe(key, current); - Expected.Add((key, observed?.Descriptor.SchemaId, observed?.StoredAt)); + Expected.Add((key, observed?.StoredSchemaId, observed?.StoredAt)); } public void ExpectUnchanged() @@ -1967,13 +1966,17 @@ namespace GameCult.Caching // Read under the gate, by a store judging a merge. internal bool HoldsVariants => _variantKeys.Count > 0; - // A store wrote its whole view: a plain record it wrote carries the ids it held, so it is no longer in-memory only. A - // variant is not cleared here: the store wrote the delta it was handed, and only a write of the variant changes that. - internal void IdsPersisted(IEnumerable keys) => Held(() => + // A store wrote its whole view: every record it wrote is stored under its type's own id, and a plain record carries the + // ids it held, so it is no longer in-memory only. A variant's ids are not cleared here: the store wrote the delta it was + // handed, and only a write of the variant changes that. + internal void WroteWholeView(IEnumerable keys) => Held(() => { foreach (var key in keys) { - if (_entries.TryGetValue(key, out var entry) && entry.Variant == null) + if (!_entries.TryGetValue(key, out var entry)) + continue; + entry.StoredSchemaId = entry.Descriptor.SchemaId; + if (entry.Variant == null) entry.IdsInMemoryOnly = false; } @@ -2565,9 +2568,9 @@ namespace GameCult.Caching if (request.HasConditions && _stores.Count > 0) throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes."); var inMemory = _entries.Values - .Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt }) + .Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.StoredSchemaId, StoredAt = entry.StoredAt }) .ToArray(); - return request.ConditionsHold(inMemory, _entries.Values, Registry.CanonicalSchemaId) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; }); }); } @@ -3487,7 +3490,10 @@ namespace GameCult.Caching descriptor, new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides.Where(Has).ToArray()), null, - Cache?.Codec); + Cache?.Codec) + { + StoredSchemaId = record.SchemaId + }; } var document = deserializePayload(resolution.Descriptor.DocumentType, record.Payload); @@ -3495,7 +3501,10 @@ namespace GameCult.Caching new CultRecordKey(record.Key), record.StoredAt, resolution.Descriptor, - document); + document) + { + StoredSchemaId = record.SchemaId + }; } protected void SetLastSchemaMigrationReports(IEnumerable reports) @@ -3561,7 +3570,7 @@ namespace GameCult.Caching var loaded = persisted.Values .Where(stored => !Entries.TryGetValue(stored.Key.Value, out var existing) || existing.StoredAt != stored.StoredAt || - existing.Descriptor.SchemaId != stored.Descriptor.SchemaId) + existing.StoredSchemaId != stored.StoredSchemaId) .ToArray(); var dropped = Entries.Values.Where(existing => !persisted.ContainsKey(existing.Key.Value)).ToArray(); if (loaded.Length > 0 || dropped.Length > 0) @@ -3612,7 +3621,7 @@ namespace GameCult.Caching Entries.Values.Any(entry => entry.HoldsIds), existingHeader: null, wholeStore: true); - Cache?.IdsPersisted(Entries.Keys); + WroteWholeView(Entries.Keys); } MarkFlushSucceeded(); @@ -3632,7 +3641,7 @@ namespace GameCult.Caching return CultCommitOutcome.Contended; var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); - if (!request.ConditionsHold(disk.Records, Entries.Values, Registry.CanonicalSchemaId)) + if (!request.ConditionsHold(disk.Records, Entries.Values)) return CultCommitOutcome.Mismatch; // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, @@ -3661,16 +3670,24 @@ namespace GameCult.Caching .Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())) .ToArray(); WriteSnapshot(records.Values, registered, ontoDisk ? disk.SchemaCatalog : Array.Empty(), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); - if (!ontoDisk) - Cache?.IdsPersisted(records.Keys); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) Entries[entry.Key.Value] = entry; + if (!ontoDisk) + WroteWholeView(records.Keys); MarkFlushSucceeded(); return CultCommitOutcome.Committed; } + // The file now holds every record of this store's view under its type's own id: the store's entries and the cache's say so. + private void WroteWholeView(IEnumerable keys) + { + foreach (var entry in Entries.Values) + entry.StoredSchemaId = entry.Descriptor.SchemaId; + Cache?.WroteWholeView(keys); + } + // Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote // (read from the record's header, never its payload), or one this cache holds. A plain-only store never decodes // another writer's records, so a schema this cache does not register cannot fail a commit that involves no variant. @@ -3689,7 +3706,7 @@ namespace GameCult.Caching { if (landing.Contains(record.Key) || removing.Contains(record.Key)) continue; - if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == record.SchemaId) + if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.StoredSchemaId == record.SchemaId) continue; arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _)); } diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs index 5d9fe4b0..e6b5c3e2 100644 --- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -5,6 +5,7 @@ using System.Linq; using GameCult.Caching.MessagePack; using MessagePack; using NUnit.Framework; +using static GameCult.Caching.Tests.EmittedDocumentTypes; namespace GameCult.Caching.Tests { @@ -15,7 +16,6 @@ namespace GameCult.Caching.Tests { private const string OldId = "old.id"; private const string OwnerId = "sha256:86186abf7d63d8a326c725d39a10f3a4db11b1e8698e8c0fcf6d9720031bc480"; - private const string SharedId = "shared.id"; [CultDocument("tests.compat_deck", "tests.compat_deck.v2", CompatibleSchemaIds = new[] { OldId, OldId })] [MessagePackObject] @@ -24,13 +24,9 @@ namespace GameCult.Caching.Tests [Key(0)] [CultName] public string Name { get; set; } = ""; } - // The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing. - [CultDocument("tests.compat_deck", "tests.compat_deck.v2")] - [MessagePackObject] - public sealed class BareDeck - { - [Key(0)] [CultName] public string Name { get; set; } = ""; - } + // The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing. Emitted, because a + // declared type that differs from DeclaringDeck only in its declaration is refused beside it by the shared registry. + private static readonly Type BareDeck = Emit("BareDeck", "tests.compat_deck", "tests.compat_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) }); [CultDocument("tests.compat_owner", "tests.compat_owner.v1", CompatibleSchemaIds = new[] { OwnerId, " " })] [MessagePackObject] @@ -39,22 +35,24 @@ namespace GameCult.Caching.Tests [Key(0)] [CultName] public string Name { get; set; } = ""; } - [CultDocument("tests.compat_claimant", "tests.compat_claimant.v1", CompatibleSchemaIds = new[] { OwnerId, SharedId })] + // One schema before and after a member was added, with nothing declared: the later type reads the earlier records by name. + [CultDocument("tests.compat_evolving", "tests.compat_evolving.v1")] [MessagePackObject] - public sealed class Claimant + public sealed class EvolvingV1 { [Key(0)] [CultName] public string Name { get; set; } = ""; } - [CultDocument("tests.compat_second_claimant", "tests.compat_second_claimant.v1", CompatibleSchemaIds = new[] { SharedId })] + [CultDocument("tests.compat_evolving", "tests.compat_evolving.v2")] [MessagePackObject] - public sealed class SecondClaimant + public sealed class EvolvingV2 { [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public string Extra { get; set; } = ""; } private static readonly CultDocumentRegistry Declaring = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaringDeck) }); - private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { typeof(BareDeck) }); + private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { BareDeck }); private static readonly CultRecordKey D = new("d"); private string _directory = ""; @@ -78,21 +76,31 @@ namespace GameCult.Caching.Tests private static CultPersistedStoreSnapshot Read(string path) => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + private static object Deck(string name) => New(BareDeck, ("Name", name)); + + private static void Rewrite(string path, Action edit) + { + var snapshot = Read(path); + edit(snapshot); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + // A store written by a runtime that knew the schema under an older id and another name: one entry that owns OldId, one // record under it. Nothing but the declaration lets this runtime's type claim it. private string OldStore(string name) { var path = Path.Combine(_directory, name); using (var cache = Open(path, Bare)) - cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); - var snapshot = Read(path); - var entry = snapshot.SchemaCatalog.Single(); - entry.SchemaId = OldId; - entry.SchemaName = "tests.legacy_deck"; - entry.ContentHash = "stale"; - entry.CompatibleSchemaIds = new[] { OldId }; - snapshot.Records.Single().SchemaId = OldId; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OldId; + entry.SchemaName = "tests.legacy_deck"; + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { OldId }; + snapshot.Records.Single().SchemaId = OldId; + }); return path; } @@ -102,16 +110,23 @@ namespace GameCult.Caching.Tests { var path = Path.Combine(_directory, name); using (var cache = Open(path, Bare)) - cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); - var snapshot = Read(path); - var entry = snapshot.SchemaCatalog.Single(); - entry.ContentHash = "stale"; - entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; - snapshot.Records.Single().SchemaId = OldId; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; + snapshot.Records.Single().SchemaId = OldId; + }); return path; } + [Test] + public void TheEmittedBareDeckIsTheDeclaringDecksSchema() + { + Assert.That(Bare.GetRequired(BareDeck).SchemaId, Is.EqualTo(Declaring.GetRequired().SchemaId)); + } + [Test] public void ARecordUnderADeclaredOldIdOpensAndACommitRewritesItUnderTheRegisteredId() { @@ -144,14 +159,15 @@ namespace GameCult.Caching.Tests { var path = Path.Combine(_directory, "renamed.cc"); using (var cache = Open(path, Bare)) - cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); - var snapshot = Read(path); - snapshot.SchemaCatalog.Single().SchemaName = "tests.renamed_deck"; - snapshot.SchemaCatalog.Single().ContentHash = "stale"; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + snapshot.SchemaCatalog.Single().SchemaName = "tests.renamed_deck"; + snapshot.SchemaCatalog.Single().ContentHash = "stale"; + }); using var reopened = Open(path, Bare); - Assert.That(reopened.Get(D)!.Name, Is.EqualTo("d")); + Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("d")); } [Test] @@ -214,13 +230,13 @@ namespace GameCult.Caching.Tests var refusal = Assert.Throws(() => cache.Commit(batch => { batch.Expect(new CultRecordKey("e"), null); - batch.Upsert(typeof(BareDeck), new BareDeck { Name = "e" }, new CultRecordKey("e")); + batch.Upsert(BareDeck, Deck("e"), new CultRecordKey("e")); }))!; Assert.That(refusal.SchemaId, Is.EqualTo(OldId)); Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); } - // A record under a declared old id is the same record the cache holds under the registered id: a condition naming it holds. + // A record under a declared old id is the same record the cache holds, loaded under that id: a condition naming it holds. [Test] public void AConditionOnARecordUnderADeclaredOldIdHoldsAndTheWholeStoreCountsAsUnchanged() { @@ -243,13 +259,14 @@ namespace GameCult.Caching.Tests var path = Path.Combine(_directory, "dir.cc"); var options = new CultCacheOpenOptions { Registry = Bare, UseDirectoryStore = true }; using (var cache = CultCacheMessagePack.Create(path, options)) - cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D)); - var manifest = Read(path); - var entry = manifest.SchemaCatalog.Single(); - entry.ContentHash = "stale"; - entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; - manifest.Records.Single().SchemaId = OldId; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, manifest => + { + var entry = manifest.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; + manifest.Records.Single().SchemaId = OldId; + }); using var declaring = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Declaring, UseDirectoryStore = true }); var current = declaring.Get(D)!; @@ -261,6 +278,170 @@ namespace GameCult.Caching.Tests }), Is.EqualTo(CultCommitOutcome.Committed)); } + // Conditions compare the id a record was loaded under with the file's, exactly. A record resolved by some other path than + // its type's own id is still unchanged when nothing on disk moved: by its schema name under a foreign id, by name across a + // schema change that declared nothing, or by a file entry that lists this runtime's id. + private static void AssertUnchangedConditionsHold(CultCache cache, CultRecordKey key, object current, Type type, object added) + { + Assert.That(cache.TryCommit(batch => + { + batch.Expect(key, current); + batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N"))); + }), Is.EqualTo(CultCommitOutcome.Committed), "Expect"); + Assert.That(cache.TryCommit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N"))); + }), Is.EqualTo(CultCommitOutcome.Committed), "ExpectUnchanged"); + } + + [Test] + public void AConditionOnARecordResolvedByNameUnderAForeignIdHolds() + { + var path = Path.Combine(_directory, "foreign.cc"); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = "foreign.id"; + entry.CompatibleSchemaIds = new[] { "foreign.id" }; + snapshot.Records.Single().SchemaId = "foreign.id"; + }); + + using var reader = Open(path, Declaring); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }); + } + + [Test] + public void AConditionOnARecordAnEvolvedTypeResolvedByNameHolds() + { + var path = Path.Combine(_directory, "evolved.cc"); + using (var cache = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV1) }))) + cache.Commit(batch => batch.Upsert(typeof(EvolvingV1), new EvolvingV1 { Name = "d" }, D)); + + using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV2) })); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, typeof(EvolvingV2), new EvolvingV2 { Name = "e" }); + } + + [Test] + public void AConditionOnARecordAFileEntryListingTheLocalIdResolvedHolds() + { + var path = Path.Combine(_directory, "listed.cc"); + var localId = Bare.GetRequired(BareDeck).SchemaId; + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OldId; + entry.CompatibleSchemaIds = new[] { OldId, localId }; + snapshot.Records.Single().SchemaId = OldId; + }); + + using var reader = Open(path, Bare); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, BareDeck, Deck("e")); + } + + // A whole-view commit writes every record under its type's own id, so the cache now holds each loaded record under that + // id: a later condition on one of them, plain or variant, holds against the rewritten file. + [Test] + public void AfterAWholeViewCommitConditionsOnRecordsLoadedUnderAnOldIdHold() + { + var path = Path.Combine(_directory, "restamped.cc"); + var b = new CultRecordKey("b"); + var v = new CultRecordKey("v"); + using (var cache = Open(path, Declaring)) + { + cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "b" }, b)); + cache.Commit(batch => batch.UpsertVariant(v, b)); + } + + Rewrite(path, snapshot => + { + foreach (var record in snapshot.Records) + record.SchemaId = OldId; + }); + + using var reader = Open(path, Declaring); + reader.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); + Assert.That(Read(path).Records.Select(record => record.SchemaId), Is.All.EqualTo(Declaring.GetRequired().SchemaId), + "the whole-view commit rewrote every record under the registered id"); + + foreach (var key in new[] { b, v }) + { + Assert.That(reader.TryCommit(batch => + { + batch.Expect(key, reader.Get(key)!); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "f" }, new CultRecordKey("f-" + key.Value)); + }), Is.EqualTo(CultCommitOutcome.Committed), "Expect " + key.Value); + } + + Assert.That(reader.TryCommit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "g" }, new CultRecordKey("g")); + }), Is.EqualTo(CultCommitOutcome.Committed), "ExpectUnchanged"); + } + + // A rolling deployment shares one single-file store between v1 (one member) and v2 (a second member, declaring v1's id). + // v1 reads v2's record through the entry that lists v1's id, and its whole-view commit writes it back under v1's id with + // the storedAt it loaded and without the member v1 lacks. A v2 cache that loaded the record before that holds a condition + // on bytes that are gone: the condition fails, and nothing derived from the lost member lands. Pulling then shows the + // record as v1 left it, and a condition on that holds. + [TestCase(false)] + [TestCase(true)] + public void AWholeViewRewriteByAnOlderVersionFailsANewerVersionsCondition(bool unchangedOnly) + { + var name = new Field("Name", typeof(string), 0, IsName: true); + var v1 = Emit("RollV1", "tests.compat_roll", "tests.compat_roll.v1", new[] { name }); + var v1Id = CultDocumentRegistry.ForTypes(new[] { v1 }).GetRequired(v1).SchemaId; + var v2 = Emit("RollV2", "tests.compat_roll", "tests.compat_roll.v2", new[] { name, new Field("Extra", typeof(string), 1) }, new[] { v1Id }); + CultCache OpenAs(Type type) => Open(Path.Combine(_directory, "roll.cc"), CultDocumentRegistry.ForTypes(new[] { type })); + string Extra(object? document) => (string?)EmittedDocumentTypes.Read(document!, "Extra") ?? ""; + var path = Path.Combine(_directory, "roll.cc"); + var k = new CultRecordKey("k"); + var y = new CultRecordKey("y"); + + using (var writer = OpenAs(v2)) + writer.Commit(batch => batch.Upsert(v2, New(v2, ("Name", "k"), ("Extra", "important")), k)); + var written = Read(path).Records.Single(); + + using var newer = OpenAs(v2); + var current = newer.Get(k)!; + using (var older = OpenAs(v1)) + { + Assert.That(EmittedDocumentTypes.Read(older.Get(k)!, "Name"), Is.EqualTo("k"), "v1 reads v2's record"); + older.Commit(batch => batch.Upsert(v1, New(v1, ("Name", "z")), new CultRecordKey("z"))); + older.Commit(batch => batch.Remove(new CultRecordKey("z"))); + } + + var rewritten = Read(path).Records.Single(); + Assert.That((rewritten.SchemaId, rewritten.StoredAt), Is.EqualTo((v1Id, written.StoredAt)), "v1 rewrote k under its id, keeping storedAt"); + Assert.That(rewritten.Payload, Is.Not.EqualTo(written.Payload), "and without Extra"); + + void Derive(CultCacheBatch batch, object held) + { + if (unchangedOnly) + batch.ExpectUnchanged(); + else + batch.Expect(k, held); + batch.Upsert(v2, New(v2, ("Name", "y"), ("Extra", "derived-from-" + Extra(held))), y); + } + + Assert.That(newer.TryCommit(batch => Derive(batch, current)), Is.EqualTo(CultCommitOutcome.Mismatch)); + using (var reopened = OpenAs(v2)) + { + Assert.That(reopened.Get(y), Is.Null, "nothing derived from the shed member landed"); + Assert.That(Extra(reopened.Get(k)), Is.Empty); + } + + newer.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + var pulled = newer.Get(k)!; + Assert.That(Extra(pulled), Is.Empty, "a pull replaces the record v1 rewrote"); + Assert.That(newer.TryCommit(batch => Derive(batch, pulled)), Is.EqualTo(CultCommitOutcome.Committed)); + } + [Test] public void ADeclarationIsRegisteredWithoutItsOwnIdAndItsEntryListsTheOwnIdFirst() { @@ -271,31 +452,5 @@ namespace GameCult.Caching.Tests Assert.That(descriptor.CompatibleSchemaIds, Is.Empty, "a type's own id and a blank id are not declared compatible ids"); Assert.That(descriptor.ToCatalogEntry().CompatibleSchemaIds, Is.EqualTo(new[] { OwnerId })); } - - [Test] - public void AnIdATypeOwnsResolvesToItEvenWhenAnEarlierTypeDeclaresItCompatible() - { - var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(Owner) }); - var owner = registry.GetRequired(); - var entry = owner.ToCatalogEntry(); - - Assert.That(registry.CanonicalSchemaId(OwnerId), Is.EqualTo(OwnerId)); - Assert.That(registry.ResolvePersistedSchema(OwnerId, new[] { entry }), Is.SameAs(owner)); - } - - [Test] - public void AnIdTwoTypesDeclareCompatibleResolvesToTheOneRegisteredFirst() - { - var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(SecondClaimant) }); - var first = registry.GetRequired(); - - Assert.That(registry.CanonicalSchemaId(SharedId), Is.EqualTo(first.SchemaId)); - } - - [Test] - public void AnIdNoTypeClaimsIsItsOwnCanonicalId() - { - Assert.That(Declaring.CanonicalSchemaId("nobody.id"), Is.EqualTo("nobody.id")); - } } } diff --git a/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs new file mode 100644 index 00000000..4ff40e25 --- /dev/null +++ b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs @@ -0,0 +1,56 @@ +#nullable enable +using System; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using MessagePack; + +namespace GameCult.Caching.Tests +{ + // Document types built at run time, each in its own dynamic assembly. CultDocumentRegistry.Shared discovers every + // [CultDocument] type in the loaded non-dynamic assemblies and refuses two that claim one schema id, so a fixture that + // needs such a pair (a type and its alias, two versions that share an id) emits them here instead of declaring them. + internal static class EmittedDocumentTypes + { + internal readonly record struct Field(string Name, Type Type, int Key, bool IsName = false); + + internal static Type Emit(string typeName, string schemaName, string schemaVersion, Field[]? fields = null, string[]? compatibleSchemaIds = null) + { + var assembly = AssemblyBuilder.DefineDynamicAssembly( + new AssemblyName("GameCult.Caching.EmittedDocuments." + Guid.NewGuid().ToString("N")), + AssemblyBuilderAccess.Run); + var type = assembly.DefineDynamicModule("main").DefineType(typeName, TypeAttributes.Class | TypeAttributes.Public | TypeAttributes.Sealed); + var documentConstructor = typeof(CultDocumentAttribute).GetConstructor(new[] { typeof(string), typeof(string) })!; + type.SetCustomAttribute(compatibleSchemaIds == null + ? new CustomAttributeBuilder(documentConstructor, new object[] { schemaName, schemaVersion }) + : new CustomAttributeBuilder( + documentConstructor, + new object[] { schemaName, schemaVersion }, + new[] { typeof(CultDocumentAttribute).GetProperty(nameof(CultDocumentAttribute.CompatibleSchemaIds))! }, + new object[] { compatibleSchemaIds })); + type.SetCustomAttribute(new CustomAttributeBuilder( + typeof(MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!, + new object[] { false })); + foreach (var field in fields ?? Array.Empty()) + { + var builder = type.DefineField(field.Name, field.Type, FieldAttributes.Public); + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { field.Key })); + if (field.IsName) + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultNameAttribute).GetConstructor(Type.EmptyTypes)!, Array.Empty())); + } + + return type.CreateType()!; + } + + // An instance of an emitted type with the named fields set. + internal static object New(Type type, params (string Field, object? Value)[] values) + { + var document = Activator.CreateInstance(type)!; + foreach (var (field, value) in values) + type.GetField(field)!.SetValue(document, value); + return document; + } + + internal static object? Read(object document, string field) => document.GetType().GetField(field)!.GetValue(document); + } +}