diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index cc64c3ed..df236053 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -285,7 +285,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Foreign(record, catalog, VersionOf(record.Payload)); // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a - // record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every + // record the manifest holds that the writer does not own is refused before any page is written, by the one rule every // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written) { @@ -293,7 +293,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); RefuseUndeclaredOnDisk( written, - key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Declares) : null, + key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null, key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, @@ -342,6 +342,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var indexRecord = ToContentAddressedIndexRecord(record, pagePayload); stored.StoredVersion = VersionOf(indexRecord.Payload); stored.StoredSchemaId = stored.Descriptor.SchemaId; + stored.StoredSchemaReadWhole = true; currentIndex[key] = indexRecord; WriteFileAtomically( ContentAddressedRecordPath(indexRecord), diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index bb78f421..b5563607 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -189,19 +189,26 @@ resolved is read-only (below). That serves a store written by a runtime whose sc cannot know (a Rust type's schema id is its entry type). A schema renamed under a stable id therefore opens in every runtime that holds the id. -**A cache owns exactly the schema ids its types register or declare compatible.** A record stored -under any other id that a type can read, through the catalog's schema name or an entry listing a -local id, is loaded read-only: Get and queries serve it, a write of another record copies it as -stored, and a staged write or removal of it (upsert, variant upsert, flatten, remove, commit) is -refused with the typed schema conflict before anything is staged. Declaring its id compatible on any -registered type makes it writable: the next write stores it under the writing type's own id. +**A cache owns a record when it declares the record's schema id, or when the type that reads it reads +it whole.** A type declares an id by registering it or listing it compatible. A type reads a record +whole when the catalog entry that publishes the id has the same content hash as the type's entry, or +compares compatible with it and leaves no persisted slot unread. An owned record is writable: a write, +relabel or removal is accepted and the write stores it under the writing type's own id (restamp on +write; only a record the cache writes is restamped, the rest is copied forward as stored). A record +stored under any other id that a type can read, through the catalog's schema name or an entry listing a +local id, only by dropping persisted slots is loaded read-only: Get and queries serve it, a write of +another record copies it as stored, and a staged write or removal of it (upsert, variant upsert, +flatten, remove, commit) is refused with the typed schema conflict before anything is staged. +Declaring its id compatible on any registered type makes it writable and is the consent to the loss: +the next write stores it under the writing type's own id and sheds the slots. The rule is checked again where the write lands, under the store's lock: a write or removal at a key -whose durable record is under an undeclared id, because another writer rewrote it after this cache +whose durable record is not owned, because another writer rewrote it after this cache loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can read it. This holds for every write through the store, the store's own `Push`, `Delete` and -`CommitBatch` included. A CultMesh single-file write declares the ids of the catalog entry it writes, -its own and those it lists compatible, and is refused the same way. +`CommitBatch` included. A CultMesh single-file write owns the ids of the catalog entry it writes, +its own and those it lists compatible, and an id whose publishing entry has the entry's schema name and +which the entry reads whole; it is refused the same way otherwise. A record that resolves to no local type is a foreign record, for example one of a type this build does not have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 9d942717..ffa12273 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -158,8 +158,9 @@ namespace GameCult.Caching /// not have, or one renamed under a new schema name without declaring its old id. The store keeps its bytes as the file holds /// them, a write of another record keeps it, and a write that would replace or remove it is refused with a /// . The cache never holds it. Declaring its schema id on a type claims it at the next load. - /// A record a type can read (through the catalog's schema name) but no type owns or declares is not foreign: the cache holds it - /// read-only, and the same refusal applies to a write or removal of it. + /// A record a type can read (through the catalog's schema name) but only by dropping persisted slots, and no type declares, is + /// not foreign: the cache holds it read-only, and the same refusal applies to a write or removal of it. A record a type reads + /// whole is owned and writable. /// public sealed class CultForeignRecord { @@ -512,9 +513,15 @@ namespace GameCult.Caching public CultDocumentDescriptor Descriptor { get; } // The schema id the record carries in its store: the id it was loaded under, which may be an older or foreign id that - // resolved to Descriptor, until a write of this record puts it under Descriptor's id. + // resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it). public string StoredSchemaId { get; internal set; } + // True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads + // it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots, + // before any variant override is shed; true for a record this cache constructed or wrote. Ownership at the staging half + // asks it; ConditionsHold never reads it. + internal bool StoredSchemaReadWhole { get; set; } = true; + // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. internal string StoredVersion { get; set; } @@ -557,7 +564,7 @@ namespace GameCult.Caching } internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly, CultCodec codec) => - new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId, StoredVersion = StoredVersion }; + new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId, StoredSchemaReadWhole = StoredSchemaReadWhole, StoredVersion = StoredVersion }; } public sealed class CultDocumentRegistry @@ -690,7 +697,7 @@ namespace GameCult.Caching // which a store carries untouched. Deciding this never decodes the record or compares shapes, so a store can classify what a // file holds without reading a payload. persisted is the catalog entry the record resolves through, and null when a // registered type owns the id. Claimants that cannot be told apart refuse. Branches past the listers (a local id the persisted - // entry lists, then the schema name) pick a type to read with. They confer no ownership: Declares decides what a cache may write. + // entry lists, then the schema name) pick a type to read with. They confer no ownership: Owns decides what a writer may write. internal CultDocumentDescriptor? Claimant( string schemaId, IReadOnlyCollection catalog, @@ -746,14 +753,53 @@ namespace GameCult.Caching catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); - // The one answer to whether a cache may write, relabel or remove a record stored under this id: a registered type owns it or - // declares it compatible. Read live, so a type registered later that declares the id claims its records at their next write. - internal bool Declares(string schemaId) + // The one answer to whether a writer may write, relabel or remove a record stored under this id + // (variants:ruling:drift-record-write-ownership): a registered type declares the id (owns it or lists it compatible), or the + // type that reads it reads it whole. Read live, so a type registered later that declares the id claims its records at their + // next write. A record read only by dropping slots is not owned: declaring its id compatible is the consent to the loss. + // Header only: the persisted catalog entry that publishes the id is compared with the claimant's entry, never a payload. A + // claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns nothing. + internal bool Owns(string schemaId, IReadOnlyCollection catalog) + { + if (Declared(schemaId)) + return true; + try + { + var claimant = Claimant(schemaId, catalog, out var persisted); + return claimant != null && persisted != null && ReadsWhole(persisted, claimant.ToCatalogEntry()); + } + catch (InvalidOperationException) + { + return false; + } + } + + // Owns' declared half: a registered type owns the id or lists it compatible. + internal bool Declared(string schemaId) { var indexes = _indexes; return indexes.BySchemaId.ContainsKey(schemaId) || indexes.ByCompatibleSchemaId.ContainsKey(schemaId); } + // A persisted entry a local entry reads whole: identical content, or compatible shapes that leave no persisted slot ignored. + internal static bool ReadsWhole(CultSchemaCatalogEntry persisted, CultSchemaCatalogEntry local) => + string.Equals(persisted.ContentHash, local.ContentHash, StringComparison.Ordinal) || + CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local)) is { IsCompatible: true, IgnoredExtraSlots.Length: 0 }; + + private static CanonicalSchemaShape ShapeOf(CultSchemaCatalogEntry entry) => + new( + entry.SchemaName, + entry.SchemaVersion, + entry.Members.Select(member => new CanonicalSchemaMember( + member.Slot, + member.MemberName, + member.TypeName, + member.IsReference, + member.IsMany, + member.TargetSchemaName, + member.IsName, + member.IndexAlias)).ToArray()); + private CultDocumentDescriptor RegisterDescriptor(CultDocumentDescriptor descriptor) { lock (_registrationGate) @@ -992,31 +1038,7 @@ namespace GameCult.Caching }); } - var persistedSchema = new CanonicalSchemaShape( - persisted.SchemaName, - persisted.SchemaVersion, - persisted.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var localSchema = new CanonicalSchemaShape( - local.SchemaName, - local.SchemaVersion, - local.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var comparison = CompareSchemaShapes(persistedSchema, localSchema); + var comparison = CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local.ToCatalogEntry())); if (!comparison.IsCompatible) { throw new InvalidOperationException( @@ -2085,6 +2107,7 @@ namespace GameCult.Caching if (!_entries.TryGetValue(record.Key, out var entry)) continue; entry.StoredSchemaId = record.SchemaId; + entry.StoredSchemaReadWhole = true; entry.StoredAt = record.StoredAt; entry.StoredVersion = version; if (entry.Variant == null) @@ -2111,7 +2134,7 @@ namespace GameCult.Caching // The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in // memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records - // it rewrote; a second call finds none. A selected record stored under an id no registered type declares is read-only, + // it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only, // so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand. public int MintElementIds() { @@ -3273,6 +3296,7 @@ namespace GameCult.Caching { Flattens = true, StoredSchemaId = variant.StoredSchemaId, + StoredSchemaReadWhole = variant.StoredSchemaReadWhole, StoredVersion = variant.StoredVersion }; } @@ -3773,6 +3797,7 @@ namespace GameCult.Caching { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); report = resolution.Report; + var readWhole = report.IgnoredExtraSlots.Length == 0; if (record.Variant != null) { var descriptor = resolution.Descriptor; @@ -3808,6 +3833,7 @@ namespace GameCult.Caching null) { StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, StoredVersion = version }; } @@ -3831,6 +3857,7 @@ namespace GameCult.Caching document) { StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, StoredVersion = version }; } @@ -3854,16 +3881,16 @@ namespace GameCult.Caching _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); } - // A cache never writes, relabels or removes a record stored under an id its types do not declare: the staging half of rule 3. - // Refused when the store carries a foreign record at the key, or holds one whose stored id no registered type owns or declares - // compatible (a record a type reads by the catalog's schema name or a persisted entry's list: read-only). Called for every - // record a cache write admits or evicts, before anything is staged. + // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. Refused when the store + // carries a foreign record at the key, or holds one that is not owned: its stored id is not declared by a registered type and + // the type that reads it drops persisted slots (read-only). A record read whole is owned; ToStoredDocument decided that from + // the same comparison Owns makes. Called for every record a cache write admits or evicts, before anything is staged. protected internal void RefuseUndeclared(string key) { var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); if (foreign != null) throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); - if (Entries.TryGetValue(key, out var held) && !Registry.Declares(held.StoredSchemaId)) + if (Entries.TryGetValue(key, out var held) && !Registry.Declared(held.StoredSchemaId) && !held.StoredSchemaReadWhole) throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } @@ -3932,11 +3959,11 @@ namespace GameCult.Caching internal bool HasStagedChecks => _checks.Count > 0; - // A stored record under an id the writer does not declare, named by the catalog entry that publishes the id. Header only: no - // payload is decoded. Null for a record under a declared id. + // A stored record under an id the writer does not own, named by the catalog entry that publishes the id. Header only: no + // payload is decoded. Null for a record under an owned id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( - CultPersistedRecord record, IReadOnlyCollection catalog, Func declares) => - declares(record.SchemaId) + CultPersistedRecord record, IReadOnlyCollection catalog, Func, bool> owns) => + owns(record.SchemaId, catalog) ? null : (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty); @@ -4044,17 +4071,17 @@ namespace GameCult.Caching DateTimeOffset.TryParse(than, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var earlier) && later > earlier; - // The one refusal of a write onto records stored under ids no registered type declares, carried foreign or held read-only. + // The one refusal of a write onto records the writer does not own, carried foreign or held read-only. // It names each record's key, stored id and schema name, never its payload. When the reload that refusal owes failed, the // refusal carries that failure as its inner exception. - // A cacheless writer (a CultMesh single-file write) declares only the catalog entry it writes, and has nothing to reload. + // A cacheless writer (a CultMesh single-file write) owns only what the catalog entry it writes declares or reads whole, and has nothing to reload. protected CultSchemaConflictException Overwrites(Exception? inner, (string Key, string SchemaId, string SchemaName)[] refused, bool cacheless = false) => new( $"{(refused.Length == 1 ? "Record" : "Records")} {string.Join(", ", refused.Select(record => $"'{record.Key}' (schema id '{record.SchemaId}', '{record.SchemaName}')"))} in {this}: " + (cacheless - ? "the catalog entry this write publishes neither owns this schema id nor lists it compatible, so the write never relabels or removes it; nothing was written and nothing was reloaded. " + - "List the id as compatible on the schema that should own it." - : "no registered type owns this schema id or declares it compatible, so this cache never writes, relabels or removes it. " + - "Declare the id compatible on the type that should own it.") + + ? "the catalog entry this write publishes neither declares this schema id nor reads it whole, so the write never relabels or removes it; nothing was written and nothing was reloaded. " + + "List the id as compatible on the schema that should own it to accept the loss of its unread slots." + : "no registered type owns or declares this schema id, or the type that reads it would drop persisted slots, so this cache never writes, relabels or removes it. " + + "Declare the id compatible on the type that should own it to accept the loss.") + (inner == null ? string.Empty : " Reloading what the write refused failed, so the cache is unchanged."), refused[0].SchemaId, refused.Select(record => record.SchemaName).Distinct(StringComparer.Ordinal).ToArray(), @@ -4586,27 +4613,36 @@ namespace GameCult.Caching using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares); + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, OwnsOnDisk()); return CultCommitOutcome.Committed; }); } - // A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id - // and the ids it lists compatible. The keys judged are every key staged, written, removed or checked. + // A cache owns through its registry (CultDocumentRegistry.Owns). A cacheless CultMesh write owns what the one catalog entry it + // writes declares: its own id and the ids it lists compatible, or an id whose publishing entry that entry reads whole + // (variants:ruling:mesh-writes-bound-by-declaration). The keys judged are every key staged, written, removed or checked. private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), - Cache != null - ? Registry.Declares - : id => upserts.Any(staged => - string.Equals(staged.Entry.SchemaId, id, StringComparison.Ordinal) || - staged.Entry.CompatibleSchemaIds.Contains(id, StringComparer.Ordinal))); + OwnsOnDisk(upserts)); + + // The owns predicate a write over the durable catalog asks. A cacheless writer has no registry: it owns what its one entry + // declares, or reads whole in the entry that publishes the id. + private Func, bool> OwnsOnDisk(IReadOnlyCollection? upserts = null) => + Cache != null || upserts == null + ? Registry.Owns + : (id, catalog) => upserts.Any(staged => + string.Equals(staged.Entry.SchemaId, id, StringComparison.Ordinal) || + staged.Entry.CompatibleSchemaIds.Contains(id, StringComparer.Ordinal) || + (CultDocumentRegistry.PublishedBy(id, catalog) is { } publisher && + string.Equals(publisher.SchemaName, staged.Entry.SchemaName, StringComparison.Ordinal) && + CultDocumentRegistry.ReadsWhole(publisher, staged.Entry))); // A refused key's durable version is what this store has now read of it. - private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func declares) + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func, bool> owns) { - var undeclared = UndeclaredOnDisk(durable, declares); + var undeclared = UndeclaredOnDisk(durable, owns); if (undeclared.Count == 0) return; RefuseUndeclaredOnDisk( @@ -4623,13 +4659,17 @@ namespace GameCult.Caching cacheless: Cache == null); } - // Every durable record under an id the writer does not declare, by key, named by the catalog entry that publishes its id. - private static Dictionary UndeclaredOnDisk(Durable? durable, Func declares) + // Every durable record under an id the writer does not own, by key, named by the catalog entry that publishes its id. The + // answer is decided once per distinct schema id, so a large store pays one comparison per id, not per record. + private static Dictionary UndeclaredOnDisk(Durable? durable, Func, bool> owns) { var undeclared = new Dictionary(StringComparer.Ordinal); + var answers = new Dictionary(StringComparer.Ordinal); + bool OwnsOnce(string id, IReadOnlyCollection catalog) => + answers.TryGetValue(id, out var answer) ? answer : answers[id] = owns(id, catalog); foreach (var record in durable?.Snapshot.Records ?? Array.Empty()) { - if (Undeclared(record, durable!.Snapshot.SchemaCatalog, declares) is { } carried) + if (Undeclared(record, durable!.Snapshot.SchemaCatalog, OwnsOnce) is { } carried) undeclared[record.Key] = carried; } @@ -4666,6 +4706,7 @@ namespace GameCult.Caching if (!Entries.TryGetValue(write.Record.Key, out var entry)) continue; entry.StoredSchemaId = write.Record.SchemaId; + entry.StoredSchemaReadWhole = true; entry.StoredAt = write.Record.StoredAt; entry.StoredVersion = version; } diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index db2fd193..aa1d7b57 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -249,7 +249,32 @@ namespace GameCult.Caching.Tests Assert.That(reader.MintElementIds(), Is.EqualTo(0)); } - // A pre-id record this cache reads only through the catalog's schema name is read-only, so the rewrite that would persist its + // A pre-id record this cache reads whole through the catalog's schema name is owned: the rewrite that persists its minted + // ids is accepted and stores it under this type's id. + [Test] + public void MintElementIdsRewritesAPreIdRecordUnderAnotherRuntimesIdThatItReadsWhole() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-drift.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using (var cache = Open(path)) + { + Assert.That(cache.MintElementIds(), Is.EqualTo(1)); + Assert.That(cache.GetStored(new CultRecordKey("a"))!.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "the rewrite restamped it"); + } + + Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); + } + + // A pre-id record this cache reads only by dropping a persisted slot is read-only, so the rewrite that would persist its // minted ids is refused typed, naming it, and the store is left byte for byte. [Test] public void MintElementIdsIsRefusedByAReadOnlyRecordWithMintedIds() @@ -262,6 +287,7 @@ namespace GameCult.Caching.Tests entry.SchemaId = otherRuntimeId; entry.ContentHash = otherRuntimeId; entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 99, MemberName = "Dropped", TypeName = "System.String" }).ToArray(); snapshot.Records.Single().SchemaId = otherRuntimeId; File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(path); diff --git a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs new file mode 100644 index 00000000..05fc89bb --- /dev/null +++ b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:drift-record-write-ownership: a writer owns a record stored under id X when it declares X, or when the type + // that reads X reads it whole; it then writes, relabels and removes it, and a write stores it under the type's own id. A record + // a type reads only by dropping persisted slots stays read-only until its id is declared compatible. + public class RestampOnWriteTests + { + // One schema name at three versions of its type and nothing declared: Grown reads Narrow's records whole (it adds a slot), + // and reads Wide's records only by dropping slot 3. + [CultDocument("tests.restamp_note", "tests.restamp_note.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteNarrow + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrown + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteWide + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + [Key(3)] public string Extra = string.Empty; + } + + // Grown with the lossy id declared: declaration is the consent to the loss. + [CultDocument("tests.restamp_note", "tests.restamp_note.v4", CompatibleSchemaIds = new[] { WideId })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrownDeclaring + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + private const string WideId = "sha256:f0d9ccd6e9a4a1c8b3a17f5b1ea1ce6b1fcd6a0f3ad1b28f2e1d4f7d91c0a2e3"; + + private static readonly CultDocumentRegistry NarrowRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow) }); + private static readonly CultDocumentRegistry GrownRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrown) }); + private static readonly CultDocumentRegistry WideRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteWide) }); + private static readonly CultDocumentRegistry DeclaringRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrownDeclaring) }); + private static readonly CultRecordKey Note = new("note"); + private static readonly CultRecordKey Other = new("other"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-restamp-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + public static readonly object[] StoreKinds = { false, true }; + + private CacheBackingStore Store(string path, bool directory) => + directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path); + + private CultCache Open(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry); + cache.AddBackingStore(Store(path, directory)); + return cache; + } + + // The schema id each key is stored under, read back through a fresh cache that declares everything it sees. + private string StoredIdOf(string path, bool directory, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, directory, registry); + return cache.GetStored(key)!.StoredSchemaId; + } + + private static string Fingerprint(string path) + { + using var sha = SHA256.Create(); + if (File.Exists(path)) + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + var parts = Directory.GetFiles(path, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(path, file) + ":" + Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(file)))); + return string.Join("|", parts); + } + + // Two records written by a writer that knew Narrow: the one under test and a bystander. + private string SeedNarrow(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, NarrowRegistry); + Assert.That(cache.Commit(batch => + { + batch.Upsert(new NoteNarrow { Name = "note", Value = 1 }, new CultRecordHandle(Note)); + batch.Upsert(new NoteNarrow { Name = "other", Value = 7 }, new CultRecordHandle(Other)); + }), Is.True); + return path; + } + + private string SeedWide(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, WideRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 1, Extra = "kept?" }, new CultRecordHandle(Note))), Is.True); + return path; + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadWholeIsWrittenUnderItsTypesId(bool directory) + { + var path = SeedNarrow(directory); + var narrowId = NarrowRegistry.GetRequired().SchemaId; + var grownId = GrownRegistry.GetRequired().SchemaId; + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Note), Is.EqualTo(narrowId)); + + using (var cache = Open(path, directory, GrownRegistry)) + { + Assert.That(cache.Get(Note)!.Value, Is.EqualTo(1)); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2, Colour = "red" }, new CultRecordHandle(Note))), Is.True); + } + + Assert.That(StoredIdOf(path, directory, GrownRegistry, Note), Is.EqualTo(grownId), "the written record moved to its type's id"); + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Other), Is.EqualTo(narrowId), "the bystander kept its persisted id (write set)"); + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note)!.Colour, Is.EqualTo("red")); + Assert.That(reopened.Get(Other)!.Value, Is.EqualTo(7)); + } + + [TestCaseSource(nameof(StoreKinds))] + public void RemovingADriftRecordReadWholeSucceeds(bool directory) + { + var path = SeedNarrow(directory); + using (var cache = Open(path, directory, GrownRegistry)) + Assert.That(cache.Commit(batch => batch.Remove(Note)), Is.True); + + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note), Is.Null); + Assert.That(reopened.Get(Other), Is.Not.Null); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadOnlyByDroppingASlotStaysReadOnly(bool directory) + { + var path = SeedWide(directory); + var before = Fingerprint(path); + + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null, "a read-only record is still served"); + var upsert = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note))))!; + Assert.That(upsert.RecordKey, Is.EqualTo("note")); + Assert.That(upsert.SchemaId, Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + var remove = Assert.Throws(() => cache.Commit(batch => batch.Remove(Note)))!; + Assert.That(remove.RecordKey, Is.EqualTo("note")); + Assert.That(Fingerprint(path), Is.EqualTo(before), "a refused write leaves the store as it was"); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADeclaredLossyDriftRecordIsWritable(bool directory) + { + var path = SeedWide(directory); + Assert.That(WideRegistry.GetRequired().SchemaId, Is.EqualTo(WideId), "the declaration names the lossy record's id"); + + using (var cache = Open(path, directory, DeclaringRegistry)) + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrownDeclaring { Name = "note", Value = 2 }, new CultRecordHandle(Note))), Is.True); + + Assert.That(StoredIdOf(path, directory, DeclaringRegistry, Note), Is.EqualTo(DeclaringRegistry.GetRequired().SchemaId)); + } + + // Another writer rewrites the record after this cache loaded it: ownership is judged again under the store's lock. + [TestCaseSource(nameof(StoreKinds))] + public void AnotherWritersDriftRecordIsJudgedUnderTheLock(bool directory) + { + var path = SeedNarrow(directory); + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null); + + // A writer that knows Wide replaces the record: this cache's loaded copy was read whole, the durable one is not. + using (var other = Open(path, directory, WideRegistry)) + { + Assert.That(other.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 3, Extra = "x" }, new CultRecordHandle(Note))), Is.True); + } + + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 4 }, new CultRecordHandle(Note))))!; + Assert.That(refusal.RecordKey, Is.EqualTo("note")); + Assert.That(StoredIdOf(path, directory, WideRegistry, Note), Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteRestampsADriftRecordOfItsSchema() + { + var path = SeedNarrow(directory: false); + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry); + + Assert.That(StoredIdOf(path, false, GrownRegistry, Note), Is.EqualTo(GrownRegistry.GetRequired().SchemaId)); + Assert.That(StoredIdOf(path, false, NarrowRegistry, Other), Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteOverALossyDriftRecordIsRefusedAndTheFileIsUntouched() + { + var path = SeedWide(directory: false); + var before = Fingerprint(path); + Assert.Throws(() => + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ExpectAgreesWithRestamp(bool directory) + { + var path = SeedNarrow(directory); + using var cache = Open(path, directory, GrownRegistry); + using var second = Open(path, directory, GrownRegistry); + var observed = cache.Get(Note)!; + + Assert.That(cache.Commit(batch => batch.ExpectUnchanged()), Is.True, "an unchanged drift record satisfies ExpectUnchanged"); + Assert.That(cache.Commit(batch => + { + batch.Expect(Note, observed); + batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note)); + }), Is.True, "a batch that Expects the drift record and upserts it commits and restamps"); + Assert.That(cache.Commit(batch => batch.ExpectUnchanged()), Is.True, "the writer's later ExpectUnchanged holds"); + Assert.That(second.Commit(batch => batch.ExpectUnchanged()), Is.False, "a cache loaded before the restamp sees the bytes change"); + } + + private static string VectorPath(string name) + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors", "ref-list-drift"); + if (Directory.Exists(vectors)) return Path.Combine(vectors, name); + } + + throw new DirectoryNotFoundException($"tests/vectors/ref-list-drift not found above {TestContext.CurrentContext.TestDirectory}."); + } + + // The store the 1.0.60 writer wrote: its 'list' holder compares compatible with RefListHolder, gaining reference semantics + // on a slot, and ignores no persisted slot, so RefListHolder reads it whole and may write it. + [Test] + public async Task TheCommitted1060VectorsListRecordIsWrittenUnderTheHoldersOwnId() + { + var path = Path.Combine(_directory, "vector.cc"); + File.Copy(VectorPath("v1-0-60-written.cc"), path); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RefListDriftTests.RefListTarget), typeof(RefListDriftTests.RefListHolder) }); + var holderId = registry.GetRequired().SchemaId; + var before = Snapshot(path); + Assert.That(before.Records.Single(record => record.Key == "list").SchemaId, Is.Not.EqualTo(holderId), "the vector's list is under an older id"); + + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.PullAllBackingStoresAsync(); + var holder = cache.GetAll().Single(); + Assert.That(cache.Commit(batch => batch.Upsert( + new RefListDriftTests.RefListHolder { Name = holder.Name, Items = holder.Items.Reverse().ToArray() }, + new CultRecordHandle(new CultRecordKey("list")))), Is.True); + } + + var after = Snapshot(path); + Assert.That(after.Records.Single(record => record.Key == "list").SchemaId, Is.EqualTo(holderId)); + Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == holderId).Members.Single(member => member.Slot == 1).IsReference, Is.True); + foreach (var kept in new[] { "alpha", "beta" }) + { + Assert.That(after.Records.Single(record => record.Key == kept).SchemaId, Is.EqualTo(before.Records.Single(record => record.Key == kept).SchemaId), kept + " keeps its persisted id"); + Assert.That(after.Records.Single(record => record.Key == kept).Payload, Is.EqualTo(before.Records.Single(record => record.Key == kept).Payload), kept + " keeps its bytes"); + } + + using var reopened = new CultCache(registry); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); + Assert.That(reopened.GetAll().Single().Items.Select(item => item.Key.Value).ToArray(), Is.EqualTo(new[] { "beta", "alpha" })); + } + + private static CultPersistedStoreSnapshot Snapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + } +} diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 843d60ba..0f4c81d7 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -12,7 +12,7 @@ Releases as 3.0.0 (major). - A single-file store reads v1, v2 and v3 stamps and refuses v4 and later by name; a directory store reads v4 and v5 and refuses v6 and later by name. Each judges the stamp before the body, so an unreadable body never makes a newer store look like damage. - A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). -- A write or removal at a key whose stored record is under a schema id the writer does not declare is refused with `CultSchemaConflictException`, and the file is left as it was. A cache declares through its registry; a cacheless CultMesh write declares through the catalog entry it writes. +- A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. - Registration refuses an owned object-list element type that has no `[CultElementId]` member (a public string with `[Key]`), naming the type and the list. Element ids let a write set merge a list by identity; a list of record references or plain values is a whole value and is not affected. - The abstract and protected surface for store kinds outside this library changed. On `SingleFileBackingStore`, `SerializeSnapshot` and `DeserializeSnapshot` are replaced by the abstract `SerializeStore`, `SerializeRecord`, `SerializeCatalogEntry` and `ReadStore`, and the abstract `SerializePayload(object)` becomes `SerializePayload(object document, Type documentType)`: a store encodes a document as the type it is given. On `CacheBackingStore`, `Judging` is removed, `ToStoredDocument` takes the record's version (`record, version, catalog, deserialize, out report`), and these protected members are new for a store kind to implement or call: `HeaderFor`, `HeaderForWrite`, `Foreign`, `SetForeignRecords`, `Undeclared`, `RefuseUndeclared`, `RefuseUndeclaredOnDisk`, `RefuseRemovalOfUndeclared`, `Refuse`, `SameStoredIdentity`, `VersionOf`, `StoredAtOver`, `Overwrites`, `StagedChecks`, `StageChecks` and `JudgeChecks`. `MarkFlushSucceeded` is unchanged. `CacheBackingStore.ForeignRecords` is new and public: the records a store holds under a schema id its cache does not declare. - `CultCommitRequest.Expected` is now `IReadOnlyList<(CultRecordKey Key, string? Version)>` (it was a key with a schema id and a stored-at), `CultCommitRequest.Checks` is new, and `ConditionsHold` takes `(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign)` in place of `(IReadOnlyCollection, IEnumerable)`. A store kind outside the library lists the record versions it holds and passes its foreign records.