From 0fbc12275a3dbed681c6d26e37ccc73b5bc457c9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 27 Oct 2026 03:39:02 +0100 Subject: [PATCH] Restamp on write: one ownership rule, Owns, under which a type that reads a stored record whole may write it CultDocumentRegistry.Owns replaces Declares as the one answer to whether a writer may write, relabel or remove a record under a schema id: the id is declared, or the type that reads it reads it whole (equal content hash, or compatible shapes with no persisted slot ignored). Staging asks the declared half or CultStoredDocument.StoredSchemaReadWhole; the apply halves (single file, directory, cacheless CultMesh) ask Owns over the durable catalog, once per distinct id. A record read only by dropping slots stays read-only until its id is declared compatible. Not built or run at this commit: verified on Yggdrasil at the cut's head. Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 5 +- .../Contracts/cultcache-persistence-format.md | 25 +- src/GameCult.Caching/CultCache.cs | 165 ++++++---- .../GameCult.Caching.Tests/ElementIdTests.cs | 28 +- .../RestampOnWriteTests.cs | 304 ++++++++++++++++++ unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 6 files changed, 454 insertions(+), 75 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/RestampOnWriteTests.cs diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index cc64c3ed..df236053 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -285,7 +285,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Foreign(record, catalog, VersionOf(record.Payload)); // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a - // record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every + // record the manifest holds that the writer does not own is refused before any page is written, by the one rule every // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written) { @@ -293,7 +293,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); RefuseUndeclaredOnDisk( written, - key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Declares) : null, + key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null, key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, @@ -342,6 +342,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var indexRecord = ToContentAddressedIndexRecord(record, pagePayload); stored.StoredVersion = VersionOf(indexRecord.Payload); stored.StoredSchemaId = stored.Descriptor.SchemaId; + stored.StoredSchemaReadWhole = true; currentIndex[key] = indexRecord; WriteFileAtomically( ContentAddressedRecordPath(indexRecord), diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index bb78f421..b5563607 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -189,19 +189,26 @@ resolved is read-only (below). That serves a store written by a runtime whose sc cannot know (a Rust type's schema id is its entry type). A schema renamed under a stable id therefore opens in every runtime that holds the id. -**A cache owns exactly the schema ids its types register or declare compatible.** A record stored -under any other id that a type can read, through the catalog's schema name or an entry listing a -local id, is loaded read-only: Get and queries serve it, a write of another record copies it as -stored, and a staged write or removal of it (upsert, variant upsert, flatten, remove, commit) is -refused with the typed schema conflict before anything is staged. Declaring its id compatible on any -registered type makes it writable: the next write stores it under the writing type's own id. +**A cache owns a record when it declares the record's schema id, or when the type that reads it reads +it whole.** A type declares an id by registering it or listing it compatible. A type reads a record +whole when the catalog entry that publishes the id has the same content hash as the type's entry, or +compares compatible with it and leaves no persisted slot unread. An owned record is writable: a write, +relabel or removal is accepted and the write stores it under the writing type's own id (restamp on +write; only a record the cache writes is restamped, the rest is copied forward as stored). A record +stored under any other id that a type can read, through the catalog's schema name or an entry listing a +local id, only by dropping persisted slots is loaded read-only: Get and queries serve it, a write of +another record copies it as stored, and a staged write or removal of it (upsert, variant upsert, +flatten, remove, commit) is refused with the typed schema conflict before anything is staged. +Declaring its id compatible on any registered type makes it writable and is the consent to the loss: +the next write stores it under the writing type's own id and sheds the slots. The rule is checked again where the write lands, under the store's lock: a write or removal at a key -whose durable record is under an undeclared id, because another writer rewrote it after this cache +whose durable record is not owned, because another writer rewrote it after this cache loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can read it. This holds for every write through the store, the store's own `Push`, `Delete` and -`CommitBatch` included. A CultMesh single-file write declares the ids of the catalog entry it writes, -its own and those it lists compatible, and is refused the same way. +`CommitBatch` included. A CultMesh single-file write owns the ids of the catalog entry it writes, +its own and those it lists compatible, and an id whose publishing entry has the entry's schema name and +which the entry reads whole; it is refused the same way otherwise. A record that resolves to no local type is a foreign record, for example one of a type this build does not have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 9d942717..ffa12273 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -158,8 +158,9 @@ namespace GameCult.Caching /// not have, or one renamed under a new schema name without declaring its old id. The store keeps its bytes as the file holds /// them, a write of another record keeps it, and a write that would replace or remove it is refused with a /// . The cache never holds it. Declaring its schema id on a type claims it at the next load. - /// A record a type can read (through the catalog's schema name) but no type owns or declares is not foreign: the cache holds it - /// read-only, and the same refusal applies to a write or removal of it. + /// A record a type can read (through the catalog's schema name) but only by dropping persisted slots, and no type declares, is + /// not foreign: the cache holds it read-only, and the same refusal applies to a write or removal of it. A record a type reads + /// whole is owned and writable. /// public sealed class CultForeignRecord { @@ -512,9 +513,15 @@ namespace GameCult.Caching public CultDocumentDescriptor Descriptor { get; } // The schema id the record carries in its store: the id it was loaded under, which may be an older or foreign id that - // resolved to Descriptor, until a write of this record puts it under Descriptor's id. + // resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it). public string StoredSchemaId { get; internal set; } + // True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads + // it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots, + // before any variant override is shed; true for a record this cache constructed or wrote. Ownership at the staging half + // asks it; ConditionsHold never reads it. + internal bool StoredSchemaReadWhole { get; set; } = true; + // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. internal string StoredVersion { get; set; } @@ -557,7 +564,7 @@ namespace GameCult.Caching } internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly, CultCodec codec) => - new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId, StoredVersion = StoredVersion }; + new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId, StoredSchemaReadWhole = StoredSchemaReadWhole, StoredVersion = StoredVersion }; } public sealed class CultDocumentRegistry @@ -690,7 +697,7 @@ namespace GameCult.Caching // which a store carries untouched. Deciding this never decodes the record or compares shapes, so a store can classify what a // file holds without reading a payload. persisted is the catalog entry the record resolves through, and null when a // registered type owns the id. Claimants that cannot be told apart refuse. Branches past the listers (a local id the persisted - // entry lists, then the schema name) pick a type to read with. They confer no ownership: Declares decides what a cache may write. + // entry lists, then the schema name) pick a type to read with. They confer no ownership: Owns decides what a writer may write. internal CultDocumentDescriptor? Claimant( string schemaId, IReadOnlyCollection catalog, @@ -746,14 +753,53 @@ namespace GameCult.Caching catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); - // The one answer to whether a cache may write, relabel or remove a record stored under this id: a registered type owns it or - // declares it compatible. Read live, so a type registered later that declares the id claims its records at their next write. - internal bool Declares(string schemaId) + // The one answer to whether a writer may write, relabel or remove a record stored under this id + // (variants:ruling:drift-record-write-ownership): a registered type declares the id (owns it or lists it compatible), or the + // type that reads it reads it whole. Read live, so a type registered later that declares the id claims its records at their + // next write. A record read only by dropping slots is not owned: declaring its id compatible is the consent to the loss. + // Header only: the persisted catalog entry that publishes the id is compared with the claimant's entry, never a payload. A + // claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns nothing. + internal bool Owns(string schemaId, IReadOnlyCollection catalog) + { + if (Declared(schemaId)) + return true; + try + { + var claimant = Claimant(schemaId, catalog, out var persisted); + return claimant != null && persisted != null && ReadsWhole(persisted, claimant.ToCatalogEntry()); + } + catch (InvalidOperationException) + { + return false; + } + } + + // Owns' declared half: a registered type owns the id or lists it compatible. + internal bool Declared(string schemaId) { var indexes = _indexes; return indexes.BySchemaId.ContainsKey(schemaId) || indexes.ByCompatibleSchemaId.ContainsKey(schemaId); } + // A persisted entry a local entry reads whole: identical content, or compatible shapes that leave no persisted slot ignored. + internal static bool ReadsWhole(CultSchemaCatalogEntry persisted, CultSchemaCatalogEntry local) => + string.Equals(persisted.ContentHash, local.ContentHash, StringComparison.Ordinal) || + CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local)) is { IsCompatible: true, IgnoredExtraSlots.Length: 0 }; + + private static CanonicalSchemaShape ShapeOf(CultSchemaCatalogEntry entry) => + new( + entry.SchemaName, + entry.SchemaVersion, + entry.Members.Select(member => new CanonicalSchemaMember( + member.Slot, + member.MemberName, + member.TypeName, + member.IsReference, + member.IsMany, + member.TargetSchemaName, + member.IsName, + member.IndexAlias)).ToArray()); + private CultDocumentDescriptor RegisterDescriptor(CultDocumentDescriptor descriptor) { lock (_registrationGate) @@ -992,31 +1038,7 @@ namespace GameCult.Caching }); } - var persistedSchema = new CanonicalSchemaShape( - persisted.SchemaName, - persisted.SchemaVersion, - persisted.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var localSchema = new CanonicalSchemaShape( - local.SchemaName, - local.SchemaVersion, - local.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var comparison = CompareSchemaShapes(persistedSchema, localSchema); + var comparison = CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local.ToCatalogEntry())); if (!comparison.IsCompatible) { throw new InvalidOperationException( @@ -2085,6 +2107,7 @@ namespace GameCult.Caching if (!_entries.TryGetValue(record.Key, out var entry)) continue; entry.StoredSchemaId = record.SchemaId; + entry.StoredSchemaReadWhole = true; entry.StoredAt = record.StoredAt; entry.StoredVersion = version; if (entry.Variant == null) @@ -2111,7 +2134,7 @@ namespace GameCult.Caching // The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in // memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records - // it rewrote; a second call finds none. A selected record stored under an id no registered type declares is read-only, + // it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only, // so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand. public int MintElementIds() { @@ -3273,6 +3296,7 @@ namespace GameCult.Caching { Flattens = true, StoredSchemaId = variant.StoredSchemaId, + StoredSchemaReadWhole = variant.StoredSchemaReadWhole, StoredVersion = variant.StoredVersion }; } @@ -3773,6 +3797,7 @@ namespace GameCult.Caching { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); report = resolution.Report; + var readWhole = report.IgnoredExtraSlots.Length == 0; if (record.Variant != null) { var descriptor = resolution.Descriptor; @@ -3808,6 +3833,7 @@ namespace GameCult.Caching null) { StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, StoredVersion = version }; } @@ -3831,6 +3857,7 @@ namespace GameCult.Caching document) { StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, StoredVersion = version }; } @@ -3854,16 +3881,16 @@ namespace GameCult.Caching _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); } - // A cache never writes, relabels or removes a record stored under an id its types do not declare: the staging half of rule 3. - // Refused when the store carries a foreign record at the key, or holds one whose stored id no registered type owns or declares - // compatible (a record a type reads by the catalog's schema name or a persisted entry's list: read-only). Called for every - // record a cache write admits or evicts, before anything is staged. + // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. Refused when the store + // carries a foreign record at the key, or holds one that is not owned: its stored id is not declared by a registered type and + // the type that reads it drops persisted slots (read-only). A record read whole is owned; ToStoredDocument decided that from + // the same comparison Owns makes. Called for every record a cache write admits or evicts, before anything is staged. protected internal void RefuseUndeclared(string key) { var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); if (foreign != null) throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); - if (Entries.TryGetValue(key, out var held) && !Registry.Declares(held.StoredSchemaId)) + if (Entries.TryGetValue(key, out var held) && !Registry.Declared(held.StoredSchemaId) && !held.StoredSchemaReadWhole) throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } @@ -3932,11 +3959,11 @@ namespace GameCult.Caching internal bool HasStagedChecks => _checks.Count > 0; - // A stored record under an id the writer does not declare, named by the catalog entry that publishes the id. Header only: no - // payload is decoded. Null for a record under a declared id. + // A stored record under an id the writer does not own, named by the catalog entry that publishes the id. Header only: no + // payload is decoded. Null for a record under an owned id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( - CultPersistedRecord record, IReadOnlyCollection catalog, Func declares) => - declares(record.SchemaId) + CultPersistedRecord record, IReadOnlyCollection catalog, Func, bool> owns) => + owns(record.SchemaId, catalog) ? null : (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty); @@ -4044,17 +4071,17 @@ namespace GameCult.Caching DateTimeOffset.TryParse(than, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var earlier) && later > earlier; - // The one refusal of a write onto records stored under ids no registered type declares, carried foreign or held read-only. + // The one refusal of a write onto records the writer does not own, carried foreign or held read-only. // It names each record's key, stored id and schema name, never its payload. When the reload that refusal owes failed, the // refusal carries that failure as its inner exception. - // A cacheless writer (a CultMesh single-file write) declares only the catalog entry it writes, and has nothing to reload. + // A cacheless writer (a CultMesh single-file write) owns only what the catalog entry it writes declares or reads whole, and has nothing to reload. protected CultSchemaConflictException Overwrites(Exception? inner, (string Key, string SchemaId, string SchemaName)[] refused, bool cacheless = false) => new( $"{(refused.Length == 1 ? "Record" : "Records")} {string.Join(", ", refused.Select(record => $"'{record.Key}' (schema id '{record.SchemaId}', '{record.SchemaName}')"))} in {this}: " + (cacheless - ? "the catalog entry this write publishes neither owns this schema id nor lists it compatible, so the write never relabels or removes it; nothing was written and nothing was reloaded. " + - "List the id as compatible on the schema that should own it." - : "no registered type owns this schema id or declares it compatible, so this cache never writes, relabels or removes it. " + - "Declare the id compatible on the type that should own it.") + + ? "the catalog entry this write publishes neither declares this schema id nor reads it whole, so the write never relabels or removes it; nothing was written and nothing was reloaded. " + + "List the id as compatible on the schema that should own it to accept the loss of its unread slots." + : "no registered type owns or declares this schema id, or the type that reads it would drop persisted slots, so this cache never writes, relabels or removes it. " + + "Declare the id compatible on the type that should own it to accept the loss.") + (inner == null ? string.Empty : " Reloading what the write refused failed, so the cache is unchanged."), refused[0].SchemaId, refused.Select(record => record.SchemaName).Distinct(StringComparer.Ordinal).ToArray(), @@ -4586,27 +4613,36 @@ namespace GameCult.Caching using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares); + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, OwnsOnDisk()); return CultCommitOutcome.Committed; }); } - // A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id - // and the ids it lists compatible. The keys judged are every key staged, written, removed or checked. + // A cache owns through its registry (CultDocumentRegistry.Owns). A cacheless CultMesh write owns what the one catalog entry it + // writes declares: its own id and the ids it lists compatible, or an id whose publishing entry that entry reads whole + // (variants:ruling:mesh-writes-bound-by-declaration). The keys judged are every key staged, written, removed or checked. private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), - Cache != null - ? Registry.Declares - : id => upserts.Any(staged => - string.Equals(staged.Entry.SchemaId, id, StringComparison.Ordinal) || - staged.Entry.CompatibleSchemaIds.Contains(id, StringComparer.Ordinal))); + OwnsOnDisk(upserts)); + + // The owns predicate a write over the durable catalog asks. A cacheless writer has no registry: it owns what its one entry + // declares, or reads whole in the entry that publishes the id. + private Func, bool> OwnsOnDisk(IReadOnlyCollection? upserts = null) => + Cache != null || upserts == null + ? Registry.Owns + : (id, catalog) => upserts.Any(staged => + string.Equals(staged.Entry.SchemaId, id, StringComparison.Ordinal) || + staged.Entry.CompatibleSchemaIds.Contains(id, StringComparer.Ordinal) || + (CultDocumentRegistry.PublishedBy(id, catalog) is { } publisher && + string.Equals(publisher.SchemaName, staged.Entry.SchemaName, StringComparison.Ordinal) && + CultDocumentRegistry.ReadsWhole(publisher, staged.Entry))); // A refused key's durable version is what this store has now read of it. - private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func declares) + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func, bool> owns) { - var undeclared = UndeclaredOnDisk(durable, declares); + var undeclared = UndeclaredOnDisk(durable, owns); if (undeclared.Count == 0) return; RefuseUndeclaredOnDisk( @@ -4623,13 +4659,17 @@ namespace GameCult.Caching cacheless: Cache == null); } - // Every durable record under an id the writer does not declare, by key, named by the catalog entry that publishes its id. - private static Dictionary UndeclaredOnDisk(Durable? durable, Func declares) + // Every durable record under an id the writer does not own, by key, named by the catalog entry that publishes its id. The + // answer is decided once per distinct schema id, so a large store pays one comparison per id, not per record. + private static Dictionary UndeclaredOnDisk(Durable? durable, Func, bool> owns) { var undeclared = new Dictionary(StringComparer.Ordinal); + var answers = new Dictionary(StringComparer.Ordinal); + bool OwnsOnce(string id, IReadOnlyCollection catalog) => + answers.TryGetValue(id, out var answer) ? answer : answers[id] = owns(id, catalog); foreach (var record in durable?.Snapshot.Records ?? Array.Empty()) { - if (Undeclared(record, durable!.Snapshot.SchemaCatalog, declares) is { } carried) + if (Undeclared(record, durable!.Snapshot.SchemaCatalog, OwnsOnce) is { } carried) undeclared[record.Key] = carried; } @@ -4666,6 +4706,7 @@ namespace GameCult.Caching if (!Entries.TryGetValue(write.Record.Key, out var entry)) continue; entry.StoredSchemaId = write.Record.SchemaId; + entry.StoredSchemaReadWhole = true; entry.StoredAt = write.Record.StoredAt; entry.StoredVersion = version; } diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index db2fd193..aa1d7b57 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -249,7 +249,32 @@ namespace GameCult.Caching.Tests Assert.That(reader.MintElementIds(), Is.EqualTo(0)); } - // A pre-id record this cache reads only through the catalog's schema name is read-only, so the rewrite that would persist its + // A pre-id record this cache reads whole through the catalog's schema name is owned: the rewrite that persists its minted + // ids is accepted and stores it under this type's id. + [Test] + public void MintElementIdsRewritesAPreIdRecordUnderAnotherRuntimesIdThatItReadsWhole() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-drift.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using (var cache = Open(path)) + { + Assert.That(cache.MintElementIds(), Is.EqualTo(1)); + Assert.That(cache.GetStored(new CultRecordKey("a"))!.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "the rewrite restamped it"); + } + + Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); + } + + // A pre-id record this cache reads only by dropping a persisted slot is read-only, so the rewrite that would persist its // minted ids is refused typed, naming it, and the store is left byte for byte. [Test] public void MintElementIdsIsRefusedByAReadOnlyRecordWithMintedIds() @@ -262,6 +287,7 @@ namespace GameCult.Caching.Tests entry.SchemaId = otherRuntimeId; entry.ContentHash = otherRuntimeId; entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 99, MemberName = "Dropped", TypeName = "System.String" }).ToArray(); snapshot.Records.Single().SchemaId = otherRuntimeId; File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(path); diff --git a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs new file mode 100644 index 00000000..05fc89bb --- /dev/null +++ b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:drift-record-write-ownership: a writer owns a record stored under id X when it declares X, or when the type + // that reads X reads it whole; it then writes, relabels and removes it, and a write stores it under the type's own id. A record + // a type reads only by dropping persisted slots stays read-only until its id is declared compatible. + public class RestampOnWriteTests + { + // One schema name at three versions of its type and nothing declared: Grown reads Narrow's records whole (it adds a slot), + // and reads Wide's records only by dropping slot 3. + [CultDocument("tests.restamp_note", "tests.restamp_note.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteNarrow + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrown + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteWide + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + [Key(3)] public string Extra = string.Empty; + } + + // Grown with the lossy id declared: declaration is the consent to the loss. + [CultDocument("tests.restamp_note", "tests.restamp_note.v4", CompatibleSchemaIds = new[] { WideId })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrownDeclaring + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + private const string WideId = "sha256:f0d9ccd6e9a4a1c8b3a17f5b1ea1ce6b1fcd6a0f3ad1b28f2e1d4f7d91c0a2e3"; + + private static readonly CultDocumentRegistry NarrowRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow) }); + private static readonly CultDocumentRegistry GrownRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrown) }); + private static readonly CultDocumentRegistry WideRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteWide) }); + private static readonly CultDocumentRegistry DeclaringRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrownDeclaring) }); + private static readonly CultRecordKey Note = new("note"); + private static readonly CultRecordKey Other = new("other"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-restamp-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + public static readonly object[] StoreKinds = { false, true }; + + private CacheBackingStore Store(string path, bool directory) => + directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path); + + private CultCache Open(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry); + cache.AddBackingStore(Store(path, directory)); + return cache; + } + + // The schema id each key is stored under, read back through a fresh cache that declares everything it sees. + private string StoredIdOf(string path, bool directory, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, directory, registry); + return cache.GetStored(key)!.StoredSchemaId; + } + + private static string Fingerprint(string path) + { + using var sha = SHA256.Create(); + if (File.Exists(path)) + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + var parts = Directory.GetFiles(path, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(path, file) + ":" + Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(file)))); + return string.Join("|", parts); + } + + // Two records written by a writer that knew Narrow: the one under test and a bystander. + private string SeedNarrow(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, NarrowRegistry); + Assert.That(cache.Commit(batch => + { + batch.Upsert(new NoteNarrow { Name = "note", Value = 1 }, new CultRecordHandle(Note)); + batch.Upsert(new NoteNarrow { Name = "other", Value = 7 }, new CultRecordHandle(Other)); + }), Is.True); + return path; + } + + private string SeedWide(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, WideRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 1, Extra = "kept?" }, new CultRecordHandle(Note))), Is.True); + return path; + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadWholeIsWrittenUnderItsTypesId(bool directory) + { + var path = SeedNarrow(directory); + var narrowId = NarrowRegistry.GetRequired().SchemaId; + var grownId = GrownRegistry.GetRequired().SchemaId; + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Note), Is.EqualTo(narrowId)); + + using (var cache = Open(path, directory, GrownRegistry)) + { + Assert.That(cache.Get(Note)!.Value, Is.EqualTo(1)); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2, Colour = "red" }, new CultRecordHandle(Note))), Is.True); + } + + Assert.That(StoredIdOf(path, directory, GrownRegistry, Note), Is.EqualTo(grownId), "the written record moved to its type's id"); + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Other), Is.EqualTo(narrowId), "the bystander kept its persisted id (write set)"); + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note)!.Colour, Is.EqualTo("red")); + Assert.That(reopened.Get(Other)!.Value, Is.EqualTo(7)); + } + + [TestCaseSource(nameof(StoreKinds))] + public void RemovingADriftRecordReadWholeSucceeds(bool directory) + { + var path = SeedNarrow(directory); + using (var cache = Open(path, directory, GrownRegistry)) + Assert.That(cache.Commit(batch => batch.Remove(Note)), Is.True); + + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note), Is.Null); + Assert.That(reopened.Get(Other), Is.Not.Null); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadOnlyByDroppingASlotStaysReadOnly(bool directory) + { + var path = SeedWide(directory); + var before = Fingerprint(path); + + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null, "a read-only record is still served"); + var upsert = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note))))!; + Assert.That(upsert.RecordKey, Is.EqualTo("note")); + Assert.That(upsert.SchemaId, Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + var remove = Assert.Throws(() => cache.Commit(batch => batch.Remove(Note)))!; + Assert.That(remove.RecordKey, Is.EqualTo("note")); + Assert.That(Fingerprint(path), Is.EqualTo(before), "a refused write leaves the store as it was"); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADeclaredLossyDriftRecordIsWritable(bool directory) + { + var path = SeedWide(directory); + Assert.That(WideRegistry.GetRequired().SchemaId, Is.EqualTo(WideId), "the declaration names the lossy record's id"); + + using (var cache = Open(path, directory, DeclaringRegistry)) + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrownDeclaring { Name = "note", Value = 2 }, new CultRecordHandle(Note))), Is.True); + + Assert.That(StoredIdOf(path, directory, DeclaringRegistry, Note), Is.EqualTo(DeclaringRegistry.GetRequired().SchemaId)); + } + + // Another writer rewrites the record after this cache loaded it: ownership is judged again under the store's lock. + [TestCaseSource(nameof(StoreKinds))] + public void AnotherWritersDriftRecordIsJudgedUnderTheLock(bool directory) + { + var path = SeedNarrow(directory); + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null); + + // A writer that knows Wide replaces the record: this cache's loaded copy was read whole, the durable one is not. + using (var other = Open(path, directory, WideRegistry)) + { + Assert.That(other.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 3, Extra = "x" }, new CultRecordHandle(Note))), Is.True); + } + + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 4 }, new CultRecordHandle(Note))))!; + Assert.That(refusal.RecordKey, Is.EqualTo("note")); + Assert.That(StoredIdOf(path, directory, WideRegistry, Note), Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteRestampsADriftRecordOfItsSchema() + { + var path = SeedNarrow(directory: false); + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry); + + Assert.That(StoredIdOf(path, false, GrownRegistry, Note), Is.EqualTo(GrownRegistry.GetRequired().SchemaId)); + Assert.That(StoredIdOf(path, false, NarrowRegistry, Other), Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteOverALossyDriftRecordIsRefusedAndTheFileIsUntouched() + { + var path = SeedWide(directory: false); + var before = Fingerprint(path); + Assert.Throws(() => + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ExpectAgreesWithRestamp(bool directory) + { + var path = SeedNarrow(directory); + using var cache = Open(path, directory, GrownRegistry); + using var second = Open(path, directory, GrownRegistry); + var observed = cache.Get(Note)!; + + Assert.That(cache.Commit(batch => batch.ExpectUnchanged()), Is.True, "an unchanged drift record satisfies ExpectUnchanged"); + Assert.That(cache.Commit(batch => + { + batch.Expect(Note, observed); + batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note)); + }), Is.True, "a batch that Expects the drift record and upserts it commits and restamps"); + Assert.That(cache.Commit(batch => batch.ExpectUnchanged()), Is.True, "the writer's later ExpectUnchanged holds"); + Assert.That(second.Commit(batch => batch.ExpectUnchanged()), Is.False, "a cache loaded before the restamp sees the bytes change"); + } + + private static string VectorPath(string name) + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors", "ref-list-drift"); + if (Directory.Exists(vectors)) return Path.Combine(vectors, name); + } + + throw new DirectoryNotFoundException($"tests/vectors/ref-list-drift not found above {TestContext.CurrentContext.TestDirectory}."); + } + + // The store the 1.0.60 writer wrote: its 'list' holder compares compatible with RefListHolder, gaining reference semantics + // on a slot, and ignores no persisted slot, so RefListHolder reads it whole and may write it. + [Test] + public async Task TheCommitted1060VectorsListRecordIsWrittenUnderTheHoldersOwnId() + { + var path = Path.Combine(_directory, "vector.cc"); + File.Copy(VectorPath("v1-0-60-written.cc"), path); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RefListDriftTests.RefListTarget), typeof(RefListDriftTests.RefListHolder) }); + var holderId = registry.GetRequired().SchemaId; + var before = Snapshot(path); + Assert.That(before.Records.Single(record => record.Key == "list").SchemaId, Is.Not.EqualTo(holderId), "the vector's list is under an older id"); + + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.PullAllBackingStoresAsync(); + var holder = cache.GetAll().Single(); + Assert.That(cache.Commit(batch => batch.Upsert( + new RefListDriftTests.RefListHolder { Name = holder.Name, Items = holder.Items.Reverse().ToArray() }, + new CultRecordHandle(new CultRecordKey("list")))), Is.True); + } + + var after = Snapshot(path); + Assert.That(after.Records.Single(record => record.Key == "list").SchemaId, Is.EqualTo(holderId)); + Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == holderId).Members.Single(member => member.Slot == 1).IsReference, Is.True); + foreach (var kept in new[] { "alpha", "beta" }) + { + Assert.That(after.Records.Single(record => record.Key == kept).SchemaId, Is.EqualTo(before.Records.Single(record => record.Key == kept).SchemaId), kept + " keeps its persisted id"); + Assert.That(after.Records.Single(record => record.Key == kept).Payload, Is.EqualTo(before.Records.Single(record => record.Key == kept).Payload), kept + " keeps its bytes"); + } + + using var reopened = new CultCache(registry); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); + Assert.That(reopened.GetAll().Single().Items.Select(item => item.Key.Value).ToArray(), Is.EqualTo(new[] { "beta", "alpha" })); + } + + private static CultPersistedStoreSnapshot Snapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + } +} diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 843d60ba..0f4c81d7 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -12,7 +12,7 @@ Releases as 3.0.0 (major). - A single-file store reads v1, v2 and v3 stamps and refuses v4 and later by name; a directory store reads v4 and v5 and refuses v6 and later by name. Each judges the stamp before the body, so an unreadable body never makes a newer store look like damage. - A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). -- A write or removal at a key whose stored record is under a schema id the writer does not declare is refused with `CultSchemaConflictException`, and the file is left as it was. A cache declares through its registry; a cacheless CultMesh write declares through the catalog entry it writes. +- A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. - Registration refuses an owned object-list element type that has no `[CultElementId]` member (a public string with `[Key]`), naming the type and the list. Element ids let a write set merge a list by identity; a list of record references or plain values is a whole value and is not affected. - The abstract and protected surface for store kinds outside this library changed. On `SingleFileBackingStore`, `SerializeSnapshot` and `DeserializeSnapshot` are replaced by the abstract `SerializeStore`, `SerializeRecord`, `SerializeCatalogEntry` and `ReadStore`, and the abstract `SerializePayload(object)` becomes `SerializePayload(object document, Type documentType)`: a store encodes a document as the type it is given. On `CacheBackingStore`, `Judging` is removed, `ToStoredDocument` takes the record's version (`record, version, catalog, deserialize, out report`), and these protected members are new for a store kind to implement or call: `HeaderFor`, `HeaderForWrite`, `Foreign`, `SetForeignRecords`, `Undeclared`, `RefuseUndeclared`, `RefuseUndeclaredOnDisk`, `RefuseRemovalOfUndeclared`, `Refuse`, `SameStoredIdentity`, `VersionOf`, `StoredAtOver`, `Overwrites`, `StagedChecks`, `StageChecks` and `JudgeChecks`. `MarkFlushSucceeded` is unchanged. `CacheBackingStore.ForeignRecords` is new and public: the records a store holds under a schema id its cache does not declare. - `CultCommitRequest.Expected` is now `IReadOnlyList<(CultRecordKey Key, string? Version)>` (it was a key with a schema id and a stored-at), `CultCommitRequest.Checks` is new, and `ConditionsHold` takes `(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign)` in place of `(IReadOnlyCollection, IEnumerable)`. A store kind outside the library lists the record versions it holds and passes its foreign records.