From a7114a1f9f97ca8828c8136fdec1f08b0fd20fc6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 3 Oct 2026 03:16:49 +0200 Subject: [PATCH] C# write set: tests for copy-forward, raw catalog carry, header rule and the inverted lay-back rules New WriteSetTests (single-file and directory stores, flush and commit): another writer's write and removal survive, a store that appeared after open is kept, an unstaged in-place change is not written, a copied record keeps the catalog entry the store holds now, an entry's unknown trailing fields survive byte for byte, conditional and unconditional commits write the same bytes, the header drops only when every copied record is known id-free, v2 exactly with a variant. Mesh: a write copies neighbours forward and replaces only its key. The lay-back and whole-view tests and the flush-restamp tests are rewritten or removed with the rules they pinned. Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 6 +- src/GameCult.Caching/CultCache.cs | 33 +- .../CompatibleSchemaIdTests.cs | 107 ++---- .../GameCult.Caching.Tests/ElementIdTests.cs | 69 ++-- .../ForeignRecordTests.cs | 35 ++ .../StoreReadabilityTests.cs | 41 ++- tests/GameCult.Caching.Tests/VariantTests.cs | 121 +++++-- tests/GameCult.Caching.Tests/WriteSetTests.cs | 319 ++++++++++++++++++ .../CultMeshSingleFileReadabilityTests.cs | 14 +- .../CultMeshSingleFileStoreTests.cs | 65 ++++ 10 files changed, 642 insertions(+), 168 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/WriteSetTests.cs diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 02b11213..b974ab5c 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -304,9 +304,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) .ToArray(); + var holdsIds = written.ToDictionary(key => key, key => Entries[key].HoldsIds, StringComparer.Ordinal); var header = HeaderForWrite( currentManifest.FormatVersion, - written.Select(key => (bool?)Entries[key].HoldsIds), + holdsIds.Values.Select(holds => (bool?)holds), currentIndex.Values.Where(record => !written.Contains(record.Key)), directoryStore: true, holdsVariants: false); @@ -314,6 +315,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore .OrderBy(record => record.Key, StringComparer.Ordinal) .ToArray(), header); + foreach (var key in written) + Entries[key].StoredHoldsIds = holdsIds[key]; + DeleteUnreferencedRecordPages(currentIndex.Values); _dirtyKeys.Clear(); diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index c1cef4c2..06757f3c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -526,6 +526,11 @@ namespace GameCult.Caching // True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds). internal bool IdsInMemoryOnly { get; set; } + // Whether the stored bytes of this record hold an element id: read when a store loaded the record, before any id was minted + // for it, and set when a store wrote it. Null when not known (a variant is not read at load). HoldsIds is what the document + // holds now; this is what the store holds, so a document changed in place and never written does not change it. + internal bool? StoredHoldsIds { get; set; } + // True when some element this record persists holds a non-empty id, and the only thing a store reads to mark its header. // It is read off the stored form each time it is asked, so a store asks it as it writes: a document changed after it was // admitted, or loaded and changed in place, writes the header its content needs. A plain record persists its document; a @@ -3469,8 +3474,8 @@ namespace GameCult.Caching // The header a write leaves, for either store kind: HeaderFor decides, given what the write can see. Marked when a record it // writes holds an id; kept marked while the store carries a record it cannot show holds none (an opaque payload it writes, or - // a copied record that is not known id-free); otherwise by content. A copied record is known id-free when this store holds - // that key under the same stored identity as the file and its document holds no id. + // a copied record that is not known id-free). A copied record is known id-free when this store holds + // that key under the same stored identity as the file and knows its stored bytes hold no id (StoredHoldsIds). protected string HeaderForWrite(string? durableHeader, IEnumerable written, IEnumerable copied, bool directoryStore, bool holdsVariants) { var holds = written.ToArray(); @@ -3484,7 +3489,7 @@ namespace GameCult.Caching Entries.TryGetValue(record.Key, out var held) && held.StoredSchemaId == record.SchemaId && held.StoredAt == record.StoredAt && - !held.HoldsIds; + held.StoredHoldsIds == false; // Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into // the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it @@ -3596,7 +3601,8 @@ namespace GameCult.Caching resolution.Descriptor, document) { - StoredSchemaId = record.SchemaId + StoredSchemaId = record.SchemaId, + StoredHoldsIds = CultElementIds.Holds(document) }; } @@ -3783,7 +3789,7 @@ namespace GameCult.Caching JudgeMerge(Array.Empty(), Array.Empty(), disk ?? new CultPersistedStoreSnapshot()); var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); ApplyWriteSet(disk, upserts, removals); - Wrote(upserts.Select(staged => staged.Record)); + Wrote(upserts); } _staged.Clear(); @@ -3817,7 +3823,7 @@ namespace GameCult.Caching Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) Entries[entry.Key.Value] = entry; - Wrote(upserts.Select(staged => staged.Record)); + Wrote(upserts); _staged.Clear(); MarkFlushSucceeded(); return CultCommitOutcome.Committed; @@ -3933,18 +3939,19 @@ namespace GameCult.Caching // The file now holds every record written, under the id and at the storedAt the write gave it: the store's entries and the // cache's say so. A copied record is on disk as the file had it, so nothing learns anything new about it: in particular the // ids it minted at load are still in memory only. - private void Wrote(IEnumerable written) + private void Wrote(IEnumerable written) { - var records = written.ToArray(); - foreach (var record in records) + var staged = written.ToArray(); + foreach (var write in staged) { - if (!Entries.TryGetValue(record.Key, out var entry)) + if (!Entries.TryGetValue(write.Record.Key, out var entry)) continue; - entry.StoredSchemaId = record.SchemaId; - entry.StoredAt = record.StoredAt; + entry.StoredSchemaId = write.Record.SchemaId; + entry.StoredAt = write.Record.StoredAt; + entry.StoredHoldsIds = write.HoldsIds; } - Cache?.Wrote(records); + Cache?.Wrote(staged.Select(write => write.Record)); } // Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs index f00da09d..8fd1c73e 100644 --- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -181,20 +181,26 @@ namespace GameCult.Caching.Tests Is.EqualTo(new[] { (D.Value, OldId, "tests.legacy_deck") })); } - // An unconditional commit writes the cache's whole view: the record the batch does not name is stamped with the registered - // id too, because the cache stamped it when it loaded it. + // An unconditional commit writes what it staged: the record the batch does not name is copied as the file holds it, under + // the old id and with the entry that publishes that id, though the cache resolved it to the registered type. [Test] - public void ACommitThatWritesTheWholeViewStampsEveryLoadedRecordWithTheRegisteredId() + public void AnUnconditionalCommitCopiesALoadedRecordUnderItsOldIdAsTheFileHoldsIt() { - var path = OldStore("whole-view.cc"); + var path = OldStore("copied.cc"); var descriptor = Declaring.GetRequired(); + var before = Read(path); using (var cache = Open(path, Declaring)) cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); var written = Read(path); - Assert.That(written.Records.Select(record => record.SchemaId), Is.All.EqualTo(descriptor.SchemaId)); - Assert.That(written.Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e" })); + Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId)); + Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId)); + var copied = written.Records.Single(record => record.Key == "d"); + var original = before.Records.Single(); + Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload))); + Assert.That(written.SchemaCatalog.Single(entry => entry.SchemaId == OldId).RawBytes, Is.EqualTo(before.SchemaCatalog.Single().RawBytes), + "the entry that published the old id is carried as its bytes"); } // A commit onto the file writes its batch and leaves every other record as the file holds it: the store writer never @@ -347,10 +353,10 @@ namespace GameCult.Caching.Tests } // A plain record and a variant loaded under a declared old id carry that id, so a condition on either holds while the file - // is unchanged. A whole-view commit then writes every record under its type's own id, and the cache holds each under that - // id: the same conditions hold against the rewritten file. + // is unchanged. An unconditional commit then copies both as they are, still under the old id, and the cache still holds + // each under it: the same conditions hold against the rewritten file. [Test] - public void ConditionsOnRecordsLoadedUnderAnOldIdHoldBeforeAndAfterAWholeViewCommit() + public void ConditionsOnRecordsLoadedUnderAnOldIdHoldBeforeAndAfterAnUnconditionalCommit() { var path = Path.Combine(_directory, "restamped.cc"); var b = new CultRecordKey("b"); @@ -391,8 +397,8 @@ namespace GameCult.Caching.Tests "commits onto the file left the loaded records under the old id"); reader.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); - Assert.That(Read(path).Records.Select(record => record.SchemaId), Is.All.EqualTo(Declaring.GetRequired().SchemaId), - "the whole-view commit rewrote every record under the registered id"); + Assert.That(Read(path).Records.Where(record => record.Key is "b" or "v").Select(record => record.SchemaId), Is.All.EqualTo(OldId), + "the unconditional commit copied the loaded records under the old id"); ConditionsHold("rewritten"); } @@ -436,8 +442,8 @@ namespace GameCult.Caching.Tests writer.Commit(batch => batch.Upsert(RollV2, New(RollV2, ("Name", "k"), ("Extra", "important")), RollK)); } - // A whole-view write by a cache of the given version, through a commit or a plain flush, that leaves the key set as it was. - private void WholeViewWrite(Type version, bool viaFlush) + // A write by a cache of the given version, through a commit or a plain flush, that leaves the key set as it was. + private void UnrelatedWrite(Type version, bool viaFlush) { using var cache = OpenRoll(version); Assert.That(EmittedDocumentTypes.Read(cache.Get(RollK)!, "Name"), Is.EqualTo("k"), $"{version.Name} reads k"); @@ -461,18 +467,18 @@ namespace GameCult.Caching.Tests batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "derived-from-" + Extra(held))), RollY); }); - // v1 reads v2's record through the entry that lists v1's id: an id v1 does not own. Its whole-view write lays k back exactly - // as v2 stored it, the member v1 lacks included, under v2's id and storedAt, so a v2 cache's condition on k still holds and - // what it derives from that member lands. + // v1 reads v2's record through the entry that lists v1's id: an id v1 does not own. A write of another record by v1 copies k + // exactly as v2 stored it, the member v1 lacks included, under v2's id and storedAt, so a v2 cache's condition on k still holds + // and what it derives from that member lands. [Test] - public void AWholeViewWriteByAnOlderVersionLeavesANewerRecordAsStored([Values] bool unchangedOnly, [Values] bool viaFlush) + public void AnUnrelatedWriteByAnOlderVersionLeavesANewerRecordAsStored([Values] bool unchangedOnly, [Values] bool viaFlush) { WriteImportant(); var written = Read(RollPath); using var newer = OpenRoll(RollV2); var current = newer.Get(RollK)!; - WholeViewWrite(RollV1, viaFlush); + UnrelatedWrite(RollV1, viaFlush); var rewritten = Read(RollPath); var record = rewritten.Records.Single(); @@ -494,9 +500,9 @@ namespace GameCult.Caching.Tests older.Commit(batch => batch.Upsert(RollV1, older.Get(RollK)!, RollK)); } - // A-B-A: v1 writes k and sheds Extra, then another v2 cache (a restart is enough) loads k under v1's id and writes its whole - // view, putting k back under v2's id: the id the first v2 cache loaded k under, with bytes that no longer hold Extra. That - // cache's condition must still fail, through commits and through plain flushes alike. + // v1 writes k and sheds Extra, then another v2 cache (a restart is enough) loads k under v1's id and writes another record. + // It copies k as v1 left it, under v1's id, so the first v2 cache's condition on k fails, through commits and through plain + // flushes alike. [TestCase(false, false)] [TestCase(true, false)] [TestCase(false, true)] @@ -510,10 +516,10 @@ namespace GameCult.Caching.Tests OlderWritesK(); Assert.That(Read(RollPath).Records.Single().SchemaId, Is.EqualTo(RollV1Id), "v1 wrote k under its id"); - WholeViewWrite(RollV2, viaFlush); - var restored = Read(RollPath).Records.Single(); - Assert.That(restored.SchemaId, Is.EqualTo(written.SchemaId), "the second v2 cache put k back under v2's id"); - Assert.That(restored.Payload, Is.Not.EqualTo(written.Payload), "without Extra"); + UnrelatedWrite(RollV2, viaFlush); + var copied = Read(RollPath).Records.Single(); + Assert.That(copied.SchemaId, Is.EqualTo(RollV1Id), "the second v2 cache copied k under v1's id"); + Assert.That(copied.Payload, Is.Not.EqualTo(written.Payload), "without Extra"); Assert.That(Derive(held, current, unchangedOnly), Is.EqualTo(CultCommitOutcome.Mismatch)); using var reopened = OpenRoll(RollV2); @@ -521,62 +527,17 @@ namespace GameCult.Caching.Tests Assert.That(Extra(reopened.Get(RollK)), Is.Empty); } - // A plain flush that re-encodes a record it loaded under another id stores it under its own id at a later storedAt, and the - // cache holds what the flush wrote: a condition on the record holds straight after, with no pull. - [Test] - public void AFlushRestampsTheRecordItReencodesInTheCacheThatFlushed() - { - WriteImportant(); - OlderWritesK(); - var shed = Read(RollPath).Records.Single(); - Assert.That(shed.SchemaId, Is.EqualTo(RollV1Id)); - - using var cache = OpenRoll(RollV2); - var current = cache.Get(RollK)!; - cache.BackingStores.Single().PushAll(); - - var flushed = Read(RollPath).Records.Single(); - Assert.That(flushed.SchemaId, Is.Not.EqualTo(RollV1Id), "the flush stored k under v2's id"); - Assert.That(string.CompareOrdinal(flushed.StoredAt, shed.StoredAt), Is.GreaterThan(0), "at a later storedAt"); - Assert.That(Derive(cache, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed)); - Assert.That(cache.TryCommit(batch => - { - batch.ExpectUnchanged(); - batch.Upsert(RollV2, New(RollV2, ("Name", "w")), new CultRecordKey("w")); - }), Is.EqualTo(CultCommitOutcome.Committed)); - } - - // A whole-view write that stores a record under another id than it was loaded under gives it a later storedAt, even when - // the file no longer holds the record it loaded. - [Test] - public void AFlushStoringARecordUnderANewIdGivesItALaterStoredAtWhateverTheFileHolds() - { - using (var older = OpenRoll(RollV1)) - older.Commit(batch => batch.Upsert(RollV1, New(RollV1, ("Name", "k")), RollK)); - var loaded = Read(RollPath).Records.Single(); - - using var cache = OpenRoll(RollV2); - using (var older = OpenRoll(RollV1)) - older.Commit(batch => batch.Remove(RollK)); - cache.BackingStores.Single().PushAll(); - - var flushed = Read(RollPath).Records.Single(); - Assert.That(flushed.SchemaId, Is.Not.EqualTo(RollV1Id)); - Assert.That(string.CompareOrdinal(flushed.StoredAt, loaded.StoredAt), Is.GreaterThan(0)); - } - - // A record a whole-view write stores exactly as the file holds it keeps its storedAt, so a condition another cache holds - // on it survives an unrelated flush. + // A record a write copies keeps its storedAt, so a condition another cache holds on it survives an unrelated write. [TestCase(false)] [TestCase(true)] - public void AWholeViewWriteKeepsTheStoredAtOfARecordItStoresUnchanged(bool viaFlush) + public void AnUnrelatedWriteKeepsTheStoredAtOfARecordItCopies(bool viaFlush) { WriteImportant(); var written = Read(RollPath).Records.Single(); using var held = OpenRoll(RollV2); var current = held.Get(RollK)!; - WholeViewWrite(RollV2, viaFlush); + UnrelatedWrite(RollV2, viaFlush); Assert.That(Read(RollPath).Records.Single().StoredAt, Is.EqualTo(written.StoredAt)); Assert.That(Derive(held, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed)); diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index e36ca015..663fd85f 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -165,9 +165,8 @@ namespace GameCult.Caching.Tests Assert.That(first.Distinct().Count(), Is.EqualTo(first.Length)); Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before), "loading mints in memory and writes nothing"); cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("other"), new CultRecordKey("other"))); - Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "a").Payload)), Is.EqualTo(first), - "the store's first write persists the ids each record minted at load"); - Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "and the write declares that the store holds ids"); + Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before), "a write of another record leaves a's bytes as they are: its minted ids stay in memory"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "and the write declares that the store holds ids: the record it wrote holds some"); } using (var again = Open(path)) @@ -212,26 +211,6 @@ namespace GameCult.Caching.Tests Assert.That(reader.MintElementIds(), Is.EqualTo(0)); } - // The store's one record, stored as another runtime stores that schema: under an id the type does not own, published by a - // catalog entry of its own. - private static void RelabelAsOtherRuntime(string path) - { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); - var entry = snapshot.SchemaCatalog.Single(); - entry.SchemaId = "other.runtime.deck"; - entry.CompatibleSchemaIds = new[] { "other.runtime.deck" }; - snapshot.Records.Single().SchemaId = "other.runtime.deck"; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); - } - - private string UnownedStore(string name) - { - var path = PathOf(name); - WritePreIdStore(path, null, ("old", "o", 1)); - RelabelAsOtherRuntime(path); - return path; - } - [Test] public void TheByteCostOfAnIdIsMeasured() { @@ -330,7 +309,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AStagedWriteThatIsFlushedCarriesTheMarkerAndPersistsLoadMintedIds() + public void AStagedWriteThatIsFlushedCarriesTheMarkerAndLeavesLoadMintedIdsToMintElementIds() { var path = PathOf("flush.cc"); WritePreIdStore(path, null, ("a", "a", 1)); @@ -338,7 +317,7 @@ namespace GameCult.Caching.Tests { cache.UpsertAsync(typeof(IdDeck), Deck("staged"), new CultRecordKey("staged")).GetAwaiter().GetResult(); cache.FlushAllBackingStores(); - Assert.That(cache.MintElementIds(), Is.EqualTo(0), "the flush wrote the whole store, a's minted ids included"); + Assert.That(cache.MintElementIds(), Is.EqualTo(1), "the flush wrote only the staged record: a's minted ids are still in memory"); } Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); @@ -376,17 +355,21 @@ namespace GameCult.Caching.Tests } [Test] - public void AnUnconditionalWriteOfAPlainRecordMarksAStoreWhoseLoadedRecordsMintedIds() + public void AnUnconditionalWriteOfAPlainRecordDoesNotPersistOrMarkTheIdsALoadedRecordMinted() { var path = PathOf("loaded-minted.cc"); WritePreIdStore(path, null, ("a", "a", 1)); + var before = DiskRecord(path, "a"); using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) { Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); UpsertPlain(cache, "p"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "nothing the write stored holds an id"); + Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before.Payload), "a is copied as it was stored"); + cache.MintElementIds(); } - Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the write persisted a's minted ids"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "MintElementIds persisted a's minted ids"); Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "a").Payload)), Has.All.Not.Empty); } @@ -751,32 +734,42 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "a whole-store flush decides by what the store now holds"); } - // Soul's P7 and P10: a store reads the header off what it writes, when it writes it. A document changed after its admission, - // or loaded and changed in place, is marked by what it holds at the flush, in both directions. + // Soul's P7 and P10: a store reads the header off what it writes, when it writes it. A staged document changed after it was + // staged, loaded or not, is marked by what it holds at the flush, in both directions. A document changed in place and not + // staged is not written, so it marks nothing. [TestCase(false)] [TestCase(true)] - public void AFlushIsMarkedByWhatADocumentHoldsNowNotWhatItHeldWhenAdmitted(bool loaded) + public void AFlushIsMarkedByWhatAStagedDocumentHoldsNowNotWhatItHeldWhenStaged(bool loaded) { - var path = PathOf(loaded ? "mutated-loaded.cc" : "mutated-admitted.cc"); + var path = PathOf(loaded ? "mutated-loaded.cc" : "mutated-staged.cc"); var key = new CultRecordKey("d"); using (var seed = Open(path)) seed.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("d"), key)); Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + var seeded = DiskRecord(path, "d"); using var cache = Open(path); - var deck = EmptyDeck("d"); - if (loaded) - deck = cache.Get(key)!; - else - cache.Commit(batch => batch.Upsert(typeof(IdDeck), deck, key)); + var deck = loaded ? cache.Get(key)! : EmptyDeck("d"); + void Stage() => cache.UpsertAsync(typeof(IdDeck), deck, key).GetAwaiter().GetResult(); + if (!loaded) + Stage(); deck.Reels.Add(new IdReel { Id = HexA }); + if (loaded) + { + cache.BackingStores[0].PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "a change in place that was never staged is not written"); + Assert.That(DiskRecord(path, "d").Payload, Is.EqualTo(seeded.Payload)); + Stage(); + } + cache.BackingStores[0].PushAll(); - Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "an id added after admission marks the flush"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "an id added after staging marks the flush"); deck.Reels.Clear(); + Stage(); cache.BackingStores[0].PushAll(); - Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the last id taken away after admission unmarks it"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the last id taken away after staging unmarks it"); } // A store with no cache has no codec to read a variant's override values with, so it cannot see whether they hold an id: the diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 796fa0e0..d707da52 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -634,6 +634,41 @@ namespace GameCult.Caching.Tests SchemaCatalog = Read(path).SchemaCatalog.Where(entry => entry.SchemaId == schemaId).ToArray() }); + // A write of another record copies a record under an id this build does not own as the store holds it: its id, storedAt, + // bytes and the catalog entry that publishes its id. + [Test] + public void AWriteOfAnotherRecordCopiesARecordUnderAnIdThisBuildDoesNotOwn([Values] Write write) + { + var path = OtherRuntimeStore("other-runtime.cc"); + var before = Stored(path, D); + var entry = CatalogEntryBytes(path, OtherRuntimeId); + + using (var cache = Open(path, DeckOnly, directory: false)) + { + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d"), "this build reads it"); + Land(cache, write, E, DeckOf("e")); + } + + Assert.That(Stored(path, D), Is.EqualTo(before)); + Assert.That(CatalogEntryBytes(path, OtherRuntimeId), Is.EqualTo(entry)); + } + + // The entry that publishes a copied record is the one the store holds when the write lands, not the one the record loaded with. + [Test] + public void ACopiedRecordIsPublishedByTheCatalogEntryTheStoreHoldsNow([Values(Write.Flush, Write.Commit)] Write write) + { + var path = OtherRuntimeStore("redescribed.cc"); + using var cache = Open(path, DeckOnly, directory: false); + var snapshot = Read(path); + snapshot.SchemaCatalog.Single(entry => entry.SchemaId == OtherRuntimeId).SchemaVersion = "tests.foreign_deck.redescribed"; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var current = CatalogEntryBytes(path, OtherRuntimeId); + + Land(cache, write, E, DeckOf("e")); + + Assert.That(CatalogEntryBytes(path, OtherRuntimeId), Is.EqualTo(current)); + } + // A write of the record itself stores what the cache holds, under the type's own id. [Test] public void AWriteOfTheRecordItselfStoresItUnderTheTypesOwnId([Values(Write.Flush, Write.Commit)] Write write) diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs index d74460e4..1e0caef0 100644 --- a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -97,8 +97,9 @@ namespace GameCult.Caching.Tests if (reads) { store.PushAll(); - // A whole-store flush writes what the store holds: one deck with no element ids is v1, whatever the file was. - Assert.That(Header(path), Is.EqualTo("cultcache.store.v1")); + // A flush applies its staged writes (none) to the file: it cannot show that a record it did not read holds no + // element id, so the file keeps the header it carries. + Assert.That(Header(path), Is.EqualTo(Header(Path.Combine(VectorRoot(), vector)))); return; } @@ -117,7 +118,8 @@ namespace GameCult.Caching.Tests if (reads) { cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))); - Assert.That(Header(path), Is.EqualTo("cultcache.store.v1"), "an unconditional commit writes the store's whole view, which holds no element id"); + Assert.That(Header(path), Is.EqualTo(Header(Path.Combine(VectorRoot(), vector))), + "the commit wrote a record without ids and copied the file's records, which it cannot show hold none"); return; } @@ -352,13 +354,14 @@ namespace GameCult.Caching.Tests Assert.That(derived.Select(entry => entry.SchemaId), Is.EqualTo(new[] { "y" }), "an entry no record needs is not written"); } + // A registered entry publishes only the ids it owns: the caller passes it for the ids its write stages, and an id nothing + // staged sits under is published by the arrived entry that lists it, or by none. var registeredLister = Entry("r", "registered.lister", "h3", "r", "z"); var arrivedLister = Entry("a", "arrived.lister", "h4", "a", "z"); - foreach (var arrived in new[] { new[] { arrivedLister }, Array.Empty() }) - { - var derived = CultSchemaCatalogEntry.Derive(new[] { Rec("a", "z") }, new[] { registeredLister }, arrived).Single(); - Assert.That(derived.SchemaId, Is.EqualTo("r"), "with no owner, a registered entry that lists the id is chosen over an arrived one"); - } + var derivedByArrived = CultSchemaCatalogEntry.Derive(new[] { Rec("a", "z") }, new[] { registeredLister }, new[] { arrivedLister }).Single(); + Assert.That(derivedByArrived.SchemaId, Is.EqualTo("a"), "a registered entry that lists the id does not publish it"); + Assert.Throws(() => + CultSchemaCatalogEntry.Derive(new[] { Rec("a", "z") }, new[] { registeredLister }, Array.Empty())); } // Entries of one tier that tie are taken in one fixed order: the catalog does not depend on the order they arrive in. @@ -430,8 +433,7 @@ namespace GameCult.Caching.Tests Assert.That(refusal.RecordKey, Is.EqualTo("a")); } - // A file entry that names an older schema under the registered schema's id, with a record kept under an older id it lists: a - // commit onto the file, or a whole-view flush, leaves a store that reopens with the record readable and the registered entry. + // A file entry that names an older schema under the registered schema's id, with a record kept under an older id it lists. private string LegacyFile(string name) { var path = Seed(name); @@ -502,17 +504,20 @@ namespace GameCult.Caching.Tests Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the other writer's record is on disk as it wrote it"); } + // One rule for both commits: an unconditional commit onto the file copies the record under the older id as it is, and is + // refused for the same reason a conditional one is. [Test] - public void AWholeViewFlushAfterARenameWithAStableIdWritesTheRegisteredEntry() + public void AnUnconditionalCommitOntoAFileWhoseRecordSitsUnderAnIdOnlyAnArrivedEntryListsIsRefusedAndTheFileLeftAsItWas() { - var path = LegacyFile("legacy-flush.cc"); - using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry })) - cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))); + var path = LegacyFile("legacy-unconditional.cc"); + var bytes = File.ReadAllBytes(path); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); - var written = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); - Assert.That(written.SchemaCatalog.Single().ContentHash, Is.Not.EqualTo("stale")); - using var reopened = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); - Assert.That(reopened.Get(new CultRecordKey("d")), Is.Not.Null); + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))))!; + Assert.That(refusal.SchemaId, Is.EqualTo("old.id")); + Assert.That(refusal.RecordKey, Is.EqualTo("d")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); } // A record the cache does not hold, under an id only an arrived entry publishes while a registered descriptor owns that diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 7567c640..d80d2902 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -665,38 +665,105 @@ namespace GameCult.Caching.Tests Assert.That(reopened.Get(other), Is.Not.Null); } - // ---- a variant sheds overrides its type lacks when it is flushed, as a plain record sheds the member ---- + // ---- a write copies what it did not stage, and re-encodes what it stages under the type's own id ---- - [Test] - public void AFlushShedsAnOverrideOfADroppedSlotSoALaterTypeReusingTheSlotDoesNotResurrectIt() + private static readonly string DriftWideId = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }).GetRequired(typeof(DriftWide)).SchemaId; + + // The same schema name as Wide, a later version that lists Wide's id as compatible and has no Wing. + private static readonly Type DriftDeclared = EmittedDocumentTypes.Emit( + "DriftDeclared", + "tests.variant_drift", + "tests.variant_drift.v4", + new[] { new EmittedDocumentTypes.Field("Name", typeof(string), 0, IsName: true), new EmittedDocumentTypes.Field("Power", typeof(int), 1) }, + new[] { DriftWideId }); + + private static readonly CultRecordKey DriftBase = new("drift-base"); + private static readonly CultRecordKey DriftVariant = new("drift-variant"); + + private static CultPersistedStoreSnapshot ReadStore(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + private void SeedDrift(string path) { - var path = PathOf("drift.cc"); - var wide = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }); - using (var writer = Open(path, registry: wide)) + using var writer = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) })); + writer.Commit(batch => { - writer.Commit(batch => + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, DriftBase); + batch.UpsertVariant(DriftVariant, DriftBase, new[] { - batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); - batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] - { - writer.Override(nameof(DriftWide.Name), "drift variant"), - writer.Override(nameof(DriftWide.Wing), "stale wing") - }); + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Wing), "stale wing") + }); + }); + } + + // Narrow reads Wide's records through the schema name and neither owns nor declares Wide's id. A write of another record + // copies them as Wide stored them: bytes, id, storedAt, the override of the slot Narrow lacks, and the entry that publishes + // Wide's id. + [Test] + public void AWriteOfAnotherRecordLeavesAVariantItsTypeHasNotDeclaredAsStored([Values] bool viaFlush) + { + var path = PathOf(viaFlush ? "drift-untouched-flush.cc" : "drift-untouched-commit.cc"); + SeedDrift(path); + var before = ReadStore(path); + + using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) + { + Assert.That(narrow.Get(DriftVariant)!.Name, Is.EqualTo("drift variant"), "Narrow reads the variant"); + var other = new CultRecordKey("other"); + if (viaFlush) + { + narrow.UpsertAsync(typeof(DriftNarrow), new DriftNarrow { Name = "other" }, other).GetAwaiter().GetResult(); + narrow.FlushAllBackingStores(); + } + else + narrow.Commit(batch => batch.Upsert(typeof(DriftNarrow), new DriftNarrow { Name = "other" }, other)); + } + + var after = ReadStore(path); + foreach (var key in new[] { DriftBase.Value, DriftVariant.Value }) + { + var was = before.Records.Single(record => record.Key == key); + var is_ = after.Records.Single(record => record.Key == key); + Assert.That((is_.SchemaId, is_.StoredAt, is_.Payload), Is.EqualTo((was.SchemaId, was.StoredAt, was.Payload)), key); + } + + Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "the dropped slot's override is still stored"); + Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes, + Is.EqualTo(before.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes)); + } + + // A type that lists Wide's id as compatible owns Wide's records: staging one writes it under the type's own id, at a later + // storedAt, without the member the type dropped, so a later type reusing that slot does not resurrect it. + [Test] + public void AStagedWriteOfADeclaredCompatibleRecordIsReEncodedUnderTheTypesOwnIdAndShedsTheDroppedSlot() + { + var path = PathOf("drift-declared.cc"); + SeedDrift(path); + var wideBase = ReadStore(path).Records.Single(record => record.Key == DriftBase.Value); + var ownId = CultDocumentRegistry.ForTypes(new[] { DriftDeclared }).GetRequired(DriftDeclared).SchemaId; + + using (var declared = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { DriftDeclared }))) + { + declared.Commit(batch => + { + batch.Upsert(DriftDeclared, declared.Get(DriftBase)!, DriftBase); + batch.UpsertVariant(DriftVariant, DriftBase, new[] { declared.Override(DriftDeclared, "Name", "drift variant") }); }); } - using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) - narrow.BackingStores[0].PushAll(); - - Assert.That(OverriddenSlotsOn(path, "drift-variant"), Is.EqualTo(new[] { 0 }), "the dropped slot's override is gone from the flushed record"); + var written = ReadStore(path); + Assert.That(written.Records.Select(record => record.SchemaId), Is.All.EqualTo(ownId)); + Assert.That(string.CompareOrdinal(written.Records.Single(record => record.Key == DriftBase.Value).StoredAt, wideBase.StoredAt), Is.GreaterThan(0)); + Assert.That(OverriddenSlotsOn(path, DriftVariant.Value), Is.EqualTo(new[] { 0 }), "the dropped slot's override is gone from the written record"); using var reused = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftReused) })); - Assert.That(reused.Get(new CultRecordKey("drift-base"))!.Colour, Is.Empty); - Assert.That(reused.Get(new CultRecordKey("drift-variant"))!.Colour, Is.Empty, "the variant inherits its base's colour, not the old wing"); + Assert.That(reused.Get(DriftBase)!.Colour, Is.Empty); + Assert.That(reused.Get(DriftVariant)!.Colour, Is.Empty, "the variant inherits its base's colour, not the old wing"); } [Test] - public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnLoadAndShedOnTheNextFlush() + public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndStaysStoredUntilTheVariantIsWritten() { var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", CultVariantOverride.Set(99, new byte[] { 0x01 }), @@ -711,10 +778,16 @@ namespace GameCult.Caching.Tests cache.BackingStores[0].PushAll(); } - Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 })); - using var reopened = Open(path); - Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.Empty); - Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2, 99 }), "a flush that staged nothing copies the variant as stored"); + using (var reopened = Open(path)) + { + Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 })); + Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + reopened.Commit(batch => batch.UpsertVariant(new CultRecordKey("hand-variant"), new CultRecordKey("hand-base"), + new[] { reopened.Override(nameof(VariantGear.Name), "hand variant") })); + } + + Assert.That(OverriddenSlotsOn(path, "hand-variant"), Is.EqualTo(new[] { 0 }), "a write of the variant stores only what it was given"); } private static int[] OverriddenSlotsOn(string path, string key) => diff --git a/tests/GameCult.Caching.Tests/WriteSetTests.cs b/tests/GameCult.Caching.Tests/WriteSetTests.cs new file mode 100644 index 00000000..68e8a195 --- /dev/null +++ b/tests/GameCult.Caching.Tests/WriteSetTests.cs @@ -0,0 +1,319 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // Rule 1 of variants:ruling:write-set-content-version-declared-ownership: a flush, a commit and a single-document write apply + // only what the writer staged to what the store durably holds now, under its lock. Every other record and catalog entry is + // copied forward as stored. Every input goes through the cache's own write API and the real stores. + public class WriteSetTests + { + [CultDocument("tests.write_set_item", "tests.write_set_item.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsItem + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public string Note = string.Empty; + } + + [CultDocument("tests.write_set_other", "tests.write_set_other.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsOther + { + [Key(0)] + [CultName] + public string Name = string.Empty; + } + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther) }); + private static readonly string ItemId = Registry.GetRequired(typeof(WsItem)).SchemaId; + private static readonly CultRecordKey A = new("a"); + private static readonly CultRecordKey X = new("x"); + + private string _directory = string.Empty; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-write-set-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, bool directory) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory }); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + private static void Write(string path, CultPersistedStoreSnapshot snapshot) => + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + // What a record is on disk: its id, its storedAt and, in a single file, its payload (a directory manifest indexes pages). + private static string Stored(string path, string key, bool directory) + { + var record = Read(path).Records.Single(candidate => candidate.Key == key); + return $"{record.SchemaId}|{record.StoredAt}|{(directory ? string.Empty : BitConverter.ToString(record.Payload))}"; + } + + private static byte[] EntryBytes(string path, string schemaId) => + Read(path).SchemaCatalog.Single(entry => entry.SchemaId == schemaId).RawBytes!; + + private static string[] Keys(string path) => Read(path).Records.Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray(); + + private static void Seed(string path, bool directory, params string[] keys) + { + using var cache = Open(path, directory); + cache.Commit(batch => + { + foreach (var key in keys) + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-" + key, Note = "seeded" }, new CultRecordKey(key)); + }); + } + + // One staged write landed by a plain flush or by an unconditional commit. + private static void Land(CultCache cache, bool commit, Type type, object document, CultRecordKey key) + { + if (commit) + { + cache.Commit(batch => batch.Upsert(type, document, key)); + return; + } + + cache.UpsertAsync(type, document, key).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + // A loads, B writes x, A stages y and flushes or commits: x keeps B's bytes, id and storedAt. A record B removed stays removed. + [Test] + public void ALoadThenAWriteAfterAnotherWritersWriteKeepsThatWrite([Values] bool directory, [Values] bool commit) + { + var path = PathOf("keeps.cc"); + Seed(path, directory, "a"); + using var a = Open(path, directory); + using (var b = Open(path, directory)) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "by-b" }, X)); + var written = Stored(path, "x", directory); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y", Note = "by-a" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "x", "y" })); + Assert.That(Stored(path, "x", directory), Is.EqualTo(written)); + } + + [Test] + public void ARecordAnotherWriterRemovedStaysRemovedAfterALaterWrite([Values] bool directory, [Values] bool commit) + { + var path = PathOf("removed.cc"); + Seed(path, directory, "a", "x"); + using var a = Open(path, directory); + Assert.That(a.Get(X), Is.Not.Null, "A loaded x"); + using (var b = Open(path, directory)) + b.Commit(batch => batch.Remove(X)); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "y" }), "A's copy of x is not written back"); + } + + // A path with nothing at it when the cache opened, and a store there by the time it writes: the write lands on that store. + [Test] + public void AWriteOverAStoreThatAppearedAfterTheOpenKeepsItsRecords([Values] bool commit) + { + var path = PathOf("appeared.cc"); + using var a = Open(path, false); + Assert.That(File.Exists(path), Is.False); + using (var b = Open(path, false)) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "x", "y" })); + } + + // An in-place change to a loaded document is not a write: it is written when the document is staged. + [Test] + public void AnInPlaceChangeThatWasNeverStagedIsNotWrittenAndAStagedOneIs([Values] bool directory, [Values] bool commit) + { + var path = PathOf("in-place.cc"); + Seed(path, directory, "a"); + using var cache = Open(path, directory); + var item = cache.Get(A)!; + item.Note = "edited"; + + cache.FlushAllBackingStores(); + using (var check = Open(path, directory)) + Assert.That(check.Get(A)!.Note, Is.EqualTo("seeded"), "an unstaged change is not written"); + + Land(cache, commit, typeof(WsItem), item, A); + using (var check = Open(path, directory)) + Assert.That(check.Get(A)!.Note, Is.EqualTo("edited")); + } + + // B changes the durable entry for the id of a record A carries. A writes a record under another id: the entry for the id it + // did not write is B's, byte for byte, not the one A registered or loaded. + [Test] + public void ARecordCopiedForwardKeepsTheCatalogEntryTheStoreHoldsNow([Values] bool directory, [Values] bool commit) + { + var path = PathOf("entry.cc"); + Seed(path, directory, "a"); + using var cache = Open(path, directory); + var snapshot = Read(path); + snapshot.SchemaCatalog.Single(entry => entry.SchemaId == ItemId).ContentHash = "changed-by-b"; + Write(path, snapshot); + var theirs = EntryBytes(path, ItemId); + + Land(cache, commit, typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + + Assert.That(EntryBytes(path, ItemId), Is.EqualTo(theirs)); + Assert.That(Read(path).SchemaCatalog.Single(entry => entry.SchemaId == ItemId).ContentHash, Is.EqualTo("changed-by-b")); + } + + // The runtime decodes seven fields of an entry. Fields after them, and the width a value was written in, are not its to drop. + [Test] + public void ACarriedCatalogEntryKeepsItsExactBytesIncludingWhatThisRuntimeDoesNotDecode([Values] bool directory, [Values] bool commit) + { + var path = PathOf("raw.cc"); + Seed(path, directory, "a"); + var original = EntryBytes(path, ItemId); + Assert.That(original[0], Is.EqualTo(0x97), "a fixarray of the seven fields the runtime knows"); + var crafted = new byte[] { 0x99 } + .Concat(original.Skip(1)) + .Concat(new byte[] { 0xd3, 0, 0, 0, 0, 0, 0, 0, 1 }) + .Concat(new byte[] { 0xc0 }) + .ToArray(); + var bytes = File.ReadAllBytes(path); + var at = IndexOf(bytes, original); + Assert.That(at, Is.GreaterThanOrEqualTo(0)); + File.WriteAllBytes(path, bytes.Take(at).Concat(crafted).Concat(bytes.Skip(at + original.Length)).ToArray()); + + using var cache = Open(path, directory); + Assert.That(cache.Get(A), Is.Not.Null, "the entry with two unknown trailing fields reads"); + Land(cache, commit, typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + + Assert.That(IndexOf(File.ReadAllBytes(path), crafted), Is.GreaterThanOrEqualTo(0), "the entry is in the written store byte for byte"); + } + + private static int IndexOf(byte[] haystack, byte[] needle) + { + for (var start = 0; start + needle.Length <= haystack.Length; start++) + { + var same = true; + for (var index = 0; same && index < needle.Length; index++) + same = haystack[start + index] == needle[index]; + if (same) + return start; + } + + return -1; + } + + // The condition is the only difference between a conditional and an unconditional commit: one staged set, one apply. + [Test] + public void AConditionalAndAnUnconditionalCommitOfOneStagedSetOntoOneStoreWriteTheSameBytes() + { + var first = PathOf("unconditional.cc"); + var second = PathOf("conditional.cc"); + Seed(first, false, "a", "x"); + File.Copy(first, second); + + foreach (var (path, conditional) in new[] { (first, false), (second, true) }) + { + using var cache = Open(path, false); + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-s", Note = "staged" }, new CultRecordKey("s")).GetAwaiter().GetResult(); + var current = cache.Get(A)!; + Assert.That(cache.Commit(batch => + { + if (conditional) + batch.Expect(A, current); + batch.Upsert(typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + batch.Remove(X); + }), Is.True); + } + + byte[] Normalized(string path) + { + var snapshot = Read(path); + foreach (var record in snapshot.Records.Where(record => record.Key is "s" or "z")) + record.StoredAt = string.Empty; + return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + } + + Assert.That(Keys(first), Is.EqualTo(new[] { "a", "s", "z" })); + Assert.That(Normalized(second), Is.EqualTo(Normalized(first))); + } + + private static void MarkAs(string path, string header) + { + var snapshot = Read(path); + snapshot.FormatVersion = header; + Write(path, snapshot); + } + + // A marked header says the store holds an element id. A writer that copies only records it knows hold none leaves it + // unmarked (marked-by-content); one that copies a record it cannot show holds none keeps the mark. + [Test] + public void AMarkedHeaderIsDroppedOnlyWhenEveryCopiedRecordIsKnownToHoldNoId([Values] bool anotherWriterAddedARecord) + { + var path = PathOf("mark.cc"); + Seed(path, false, "a"); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + using var cache = Open(path, false); + if (anotherWriterAddedARecord) + { + using (var other = Open(path, false)) + other.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + } + + Land(cache, commit: false, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(anotherWriterAddedARecord ? CultPersistedStoreSnapshot.FormatV3 : CultPersistedStoreSnapshot.FormatV1)); + } + + // v2 exactly when the written store holds a variant: a copied variant keeps it, and the last one removed drops it. + [Test] + public void TheV2HeaderIsWrittenExactlyWhileTheStoreHoldsAVariant() + { + var path = PathOf("variant.cc"); + Seed(path, false, "a"); + var variant = new CultRecordKey("v"); + using (var cache = Open(path, false)) + { + cache.Commit(batch => batch.UpsertVariant(variant, A, new[] + { + cache.Override(nameof(WsItem.Name), "name-v"), + cache.Override(nameof(WsItem.Note), "variant") + })); + } + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV2)); + + using (var cache = Open(path, false)) + { + Land(cache, commit: true, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV2), "the variant was copied"); + cache.Commit(batch => batch.Remove(variant)); + } + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs index a0933b68..d0c45399 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs @@ -72,8 +72,20 @@ public sealed class CultMeshSingleFileReadabilityTests if (reads) { + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes); write(path); - CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Should().ContainSingle(name); + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + after.Records.Where(record => record.Key == "publication").Should().ContainSingle(name); + var others = before.Records.Where(record => record.Key != "publication").ToArray(); + after.Records.Should().HaveCount(others.Length + 1, name); + foreach (var record in others) + { + var kept = after.Records.Single(candidate => candidate.Key == record.Key); + kept.SchemaId.Should().Be(record.SchemaId, name); + kept.StoredAt.Should().Be(record.StoredAt, name); + kept.Payload.Should().Equal(record.Payload, name + " copies the file's other records as they are"); + } + continue; } diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs index ef934c8b..9a59e958 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs @@ -53,6 +53,13 @@ public sealed class CultMeshSingleFileStoreTests SchemaId = "eve.entity_soa.v1" }; + private static CultMeshBodyPublicationDocument PublicationOf(string id) => new() + { + BodyId = "aetheria:" + id, + ProducerId = "aetheria", + SchemaId = "eve.entity_soa.v1" + }; + private static string SchemaId() => CultDocumentRegistry.Shared.GetRequired().SchemaId; private static bool IsALink(string path) @@ -103,6 +110,64 @@ public sealed class CultMeshSingleFileStoreTests } } + private static CultPersistedStoreSnapshot Snapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // A Mesh write is a write set of one key: every other record in the file and the entry that publishes it are copied forward as + // the file holds them, and a second write of the key replaces only that key. + [Test] + public void AMeshWriteIntoAStoreHoldingOtherRecordsCopiesThemForwardAndReplacesOnlyItsKey() + { + var path = Path.Combine(_root, "shared.cc"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions())) + { + cache.Commit(batch => + { + batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("a"), new CultRecordKey("a")); + batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("b"), new CultRecordKey("b")); + }); + } + + var before = Snapshot(path); + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x90 }); + var after = Snapshot(path); + + after.Records.Select(record => record.Key).Should().Equal("a", "b", "publication"); + foreach (var key in new[] { "a", "b" }) + { + var was = before.Records.Single(record => record.Key == key); + var kept = after.Records.Single(record => record.Key == key); + (kept.SchemaId, kept.StoredAt).Should().Be((was.SchemaId, was.StoredAt)); + kept.Payload.Should().Equal(was.Payload); + } + + after.SchemaCatalog.Single(entry => entry.SchemaId == SchemaId()).RawBytes.Should().Equal(before.SchemaCatalog.Single(entry => entry.SchemaId == SchemaId()).RawBytes); + after.SchemaCatalog.Should().Contain(entry => entry.SchemaId == "raw:schema"); + + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x91, 0x01 }); + var again = Snapshot(path); + again.Records.Select(record => record.Key).Should().Equal("a", "b", "publication"); + again.Records.Single(record => record.Key == "publication").Payload.Should().Equal(new byte[] { 0x91, 0x01 }); + foreach (var key in new[] { "a", "b" }) + again.Records.Single(record => record.Key == key).Payload.Should().Equal(before.Records.Single(record => record.Key == key).Payload); + } + + // The writer cannot see whether the raw payload or the records it copies hold element ids, so a marked file stays marked. + [Test] + public void AMeshRawWriteIntoAMarkedStoreKeepsItMarked() + { + var path = Path.Combine(_root, "marked.cc"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions())) + cache.Commit(batch => batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("a"), new CultRecordKey("a"))); + var marked = Snapshot(path); + marked.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(marked)); + + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x90 }); + + Snapshot(path).FormatVersion.Should().Be(CultPersistedStoreSnapshot.FormatV3); + } + [Test] public void ADanglingLinkIsAnIoErrorOnTypedWriteRawWriteAndReadNeverAMissingFile() {