From c55d34fb3258cb7b02ad5e2644c982984a14397e Mon Sep 17 00:00:00 2001 From: soul Date: Tue, 10 Nov 2026 14:35:10 +0000 Subject: [PATCH 01/20] CultCache: a record whose payload carries slots past its reader's, at any depth, is held read-only The guard formatter counts each int-keyed object's slots into a thread-bound scope opened around the one payload decode; ToStoredDocument turns the finding into CultStoredDocument.UnreadSlots (explained only for depth-0 slots the catalog comparison reports ignored); HeldOwned refuses an unexplained mark whatever is declared; RefuseUndeclared throws the one refusal with UnreadSlots; CultCache.IsWritable asks the same predicate. Vectors: extra-slot-unit-arm, extra-slot-store.cc. Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 18 +- .../CultSlotScope.cs | 47 +++ .../CultUnionGuard.cs | 55 +++- .../DirectoryMessagePackBackingStore.cs | 2 +- src/GameCult.Caching/CultCache.cs | 106 +++++- .../CultSchemaConflictException.cs | 13 +- src/GameCult.Caching/CultUnreadSlots.cs | 53 +++ .../SlotReadOnlyTests.cs | 310 ++++++++++++++++++ .../UnionParityVectorTests.cs | 41 ++- tests/GameCult.Caching.Tests/VariantTests.cs | 16 +- tests/vectors/union-parity/README.md | 14 +- .../vectors/union-parity/extra-slot-store.cc | Bin 0 -> 2809 bytes .../union-parity/extra-slot-unit-arm.msgpack | 1 + tests/vectors/union-parity/outcomes.msgpack | Bin 502 -> 563 bytes 14 files changed, 644 insertions(+), 32 deletions(-) create mode 100644 src/GameCult.Caching.MessagePack/CultSlotScope.cs create mode 100644 src/GameCult.Caching/CultUnreadSlots.cs create mode 100644 tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs create mode 100644 tests/vectors/union-parity/extra-slot-store.cc create mode 100644 tests/vectors/union-parity/extra-slot-unit-arm.msgpack diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 0acee656..15b70c9c 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -166,7 +166,7 @@ public static class CultDocumentMessagePackSerialization private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers) { - // The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer + // The guard goes first and wraps whatever formatter the rest of the composite would hand out (a union, or a concrete int-keyed object for its slot count), so a consumer // resolver still encodes an arm while the key check runs. var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray(); var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray()); @@ -214,6 +214,19 @@ public static class CultDocumentMessagePackSerialization ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } + // The same decode, with the slots it read past noted (CultSlotScope). Opened around the one decode and after the depth check, + // so a payload past the nesting limit is refused, never marked. + internal static CultDecoded DeserializeMarked(Type type, byte[] payload, CultDocumentRegistry registry) + { + if (registry == null) throw new ArgumentNullException(nameof(registry)); + registry.GetRequired(type); + CultPayloadDepth.Check(payload, type, PayloadDirection.Read); + using var scope = CultSlotScope.Open(); + var document = MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type)) + ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); + return new CultDecoded(document, scope.Finding()); + } + /// /// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member /// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil. @@ -823,4 +836,7 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore { return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry); } + + internal override CultDecoded DeserializeMarked(Type documentType, byte[] payload) => + CultDocumentMessagePackSerialization.DeserializeMarked(documentType, payload, Registry); } diff --git a/src/GameCult.Caching.MessagePack/CultSlotScope.cs b/src/GameCult.Caching.MessagePack/CultSlotScope.cs new file mode 100644 index 00000000..64d5b6c6 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultSlotScope.cs @@ -0,0 +1,47 @@ +using System; +using GameCult.Caching; + +namespace GameCult.Caching.MessagePack; + +/// +/// The marks one payload decode noted: the guard's slot formatter reports every object whose array held more slots than its type +/// declares, with the nesting depth it sat at, and DeserializeMarked reads the finding out. Thread-bound and open only around the one +/// decode, so a decode outside it (a CultMesh read, a codec) notes nothing. +/// +internal sealed class CultSlotScope : IDisposable +{ + [ThreadStatic] private static CultSlotScope? current; + + private readonly CultSlotScope? previous; + private string? typeName; + private int slotCount; + private int declaredCount; + private int occurrences; + private bool allAtDepthZero = true; + + private CultSlotScope() => previous = current; + + internal static CultSlotScope? Current => current; + + internal static CultSlotScope Open() => current = new CultSlotScope(); + + // 0 for the document's own object; each object nested inside it adds one (a union adds none, its arm is the object). + internal int Depth { get; set; } + + internal void Note(string name, int slots, int declared) + { + if (occurrences++ == 0) + { + typeName = name; + slotCount = slots; + declaredCount = declared; + } + + if (Depth > 0) allAtDepthZero = false; + } + + internal CultUnreadSlots? Finding() => + occurrences == 0 ? null : new CultUnreadSlots(typeName!, slotCount, declaredCount, occurrences, allAtDepthZero, explained: false); + + public void Dispose() => current = previous; +} diff --git a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs index 2b60c80b..6fb2e004 100644 --- a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs +++ b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs @@ -8,7 +8,7 @@ using MessagePack.Formatters; namespace GameCult.Caching.MessagePack; /// -/// The one place an unknown union arm or an unnamed subtype is decided. A union is an abstract class or interface carrying +/// The one sensor of an unknown union arm, an unnamed subtype and a slot past the reader's declared slots. A union is an abstract class or interface carrying /// its own [Union] attributes (never inherited: a union whose arms are also a union has its own key set). MessagePack-CSharp /// still encodes the [key, armSlots] shape; this wraps its formatter so that a key the union does not declare is refused on /// read, and a value whose runtime type is not a declared arm is refused on write, where MessagePack alone yields null and nil. @@ -18,7 +18,58 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol public IMessagePackFormatter? GetFormatter() { var arms = typeof(T).GetCustomAttributes(inherit: false).ToArray(); - return arms.Length == 0 || inner.GetFormatter() is not { } formatter ? null : new Formatter(formatter, arms); + if (arms.Length == 0) + return DeclaredSlots(typeof(T)) is { } declared && inner.GetFormatter() is { } objectFormatter + ? new SlotFormatter(objectFormatter, declared) + : null; + return inner.GetFormatter() is not { } formatter ? null : new Formatter(formatter, arms); + } + + // The slots a concrete int-keyed [MessagePackObject] type declares (highest [Key] plus one), by MessagePack's member rule: public + // instance properties and fields, base keys included, [IgnoreMember] excluded. Null for any other type, which the guard leaves alone. + private static int? DeclaredSlots(Type type) + { + if (type.IsAbstract || type.IsInterface || type.IsGenericTypeDefinition) return null; + if (type.GetCustomAttribute(inherit: true) is not { KeyAsPropertyName: false }) return null; + var highest = -1; + const BindingFlags members = BindingFlags.Public | BindingFlags.Instance; + foreach (var member in type.GetFields(members).Cast().Concat(type.GetProperties(members))) + { + if (member.GetCustomAttribute(inherit: true) != null) continue; + if (member.GetCustomAttribute(inherit: true)?.IntKey is { } key) highest = Math.Max(highest, key); + } + + return highest + 1; + } + + // Counts a concrete object's array header against its declared slots and notes the excess into the open CultSlotScope, then lets the + // inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through. + private sealed class SlotFormatter(IMessagePackFormatter inner, int declared) : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options) => + inner.Serialize(ref writer, value, options); + + public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var scope = CultSlotScope.Current; + if (scope == null) return inner.Deserialize(ref reader, options); + var peek = reader.CreatePeekReader(); + if (!peek.TryReadNil() && peek.NextMessagePackType == MessagePackType.Array) + { + var count = peek.ReadArrayHeader(); + if (count > declared) scope.Note(typeof(T).Name, count, declared); + } + + scope.Depth++; + try + { + return inner.Deserialize(ref reader, options); + } + finally + { + scope.Depth--; + } + } } private sealed class Formatter(IMessagePackFormatter inner, UnionAttribute[] arms) : IMessagePackFormatter diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index a113db44..8f76cbae 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -636,7 +636,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore record, VersionOf(metadata.Payload), catalog, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), + (type, payload) => CultDocumentMessagePackSerialization.DeserializeMarked(type, payload, Registry), out report); // The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with. stored.StoredSchemaId = metadata.SchemaId; diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 6f04d99c..da791b82 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -516,14 +516,22 @@ namespace GameCult.Caching // resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it). public string StoredSchemaId { get; internal set; } - // True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads - // it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots, + // True when the schema this record was loaded under left none of its persisted slots unread by Descriptor and its payload + // carried none past Descriptor's, so the type reads it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots, // before any variant override is shed. A record this cache constructed or wrote is stored under Descriptor's own id, which // the registry declares, so only a loaded record under another id is ever asked. The staging half asks it; // ConditionsHold never reads it. Only the store's loaded view carries it: the cache's resolved copies and flattens are never // asked. internal bool StoredSchemaReadWhole { get; set; } + /// + /// What the store's decode of this record's payload found past the slots this build declares, at any depth; null when none. Set by + /// ToStoredDocument on the store's loaded view only (the cache's resolved copy of a variant does not carry it; ask + /// ). A mark that is not keeps the record + /// unowned even under a declared id: it cannot be written or removed (HeldOwned). + /// + public CultUnreadSlots? UnreadSlots { get; internal set; } + // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for, // so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record // this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that @@ -2656,6 +2664,17 @@ namespace GameCult.Caching return _entries.TryGetValue(key.Value, out var stored) ? stored : null; } + /// + /// Whether a write or removal of the key would be let through by every attached store: false when a store lists the key foreign, + /// or holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's, see + /// ); true for a key no store holds. Live and advisory: the write still decides. + /// + public bool IsWritable(CultRecordKey key) + { + lock (Gate()) + return _stores.All(entry => entry.Store.Refusal(key.Value, null) == null); + } + public object? Get(CultRecordKey key) { lock (Gate()) @@ -3937,11 +3956,21 @@ namespace GameCult.Caching string version, IReadOnlyCollection catalog, Func deserializePayload, + out CultSchemaMigrationReport report) => + ToStoredDocument(record, version, catalog, (type, payload) => new CultDecoded(deserializePayload(type, payload), null), out report); + + // The same, for a decode that notes the extra slots it read past (DeserializeMarked): the mark is turned into + // CultStoredDocument.UnreadSlots here, the one place that knows both the payload's marks and the catalog comparison. + internal CultStoredDocument ToStoredDocument( + CultPersistedRecord record, + string version, + IReadOnlyCollection catalog, + Func deserializePayload, out CultSchemaMigrationReport report) { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); report = resolution.Report; - var readWhole = report.IgnoredExtraSlots.Length == 0; + CultUnreadSlots? unread = null; if (record.Variant != null) { var descriptor = resolution.Descriptor; @@ -3955,13 +3984,22 @@ namespace GameCult.Caching .ToArray(); if (shed.Length > 0) { - // A slot the schema diff already reported is not reported twice. + // A slot the schema diff already reported is not reported twice. One it did not report is a mark nothing explains: + // the variant is read-only and is never rewritten. var unreported = shed.Except(report.IgnoredExtraSlots).ToArray(); + if (unreported.Length > 0) + unread = new CultUnreadSlots( + descriptor.DocumentType.Name, + unreported.Max() + 1, + descriptor.RichMembers.Select(member => member.Slot + 1).DefaultIfEmpty(0).Max(), + 1, + allAtDepthZero: true, + explained: false); report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(unreported).OrderBy(slot => slot).ToArray(); report.Warnings = report.Warnings.Concat(unreported.Select(slot => new CultSchemaMigrationWarning { Code = "ignored_extra_slot", - Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush." + Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the variant is read-only." })).ToArray(); } @@ -3977,16 +4015,17 @@ namespace GameCult.Caching null) { StoredSchemaId = record.SchemaId, - StoredSchemaReadWhole = readWhole, + StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0, + UnreadSlots = unread, StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; } - object document; + CultDecoded decoded; try { - document = deserializePayload(resolution.Descriptor.DocumentType, record.Payload); + decoded = deserializePayload(resolution.Descriptor.DocumentType, record.Payload); } catch (Exception exception) when (exception is not OperationCanceledException) { @@ -3995,14 +4034,17 @@ namespace GameCult.Caching for (var cause = exception; cause != null; cause = cause.InnerException) causes.Add(cause.Message); throw new InvalidDataException($"Record '{record.Key}' (schema '{record.SchemaId}') cannot be decoded: {string.Join(" <- ", causes.Distinct())}", exception); } + // Explained only when every mark is at depth 0 and the schema comparison reported ignored slots under this id. + unread = decoded.UnreadSlots?.Explaining(decoded.UnreadSlots.AllAtDepthZero && report.IgnoredExtraSlots.Length > 0); return new CultStoredDocument( new CultRecordKey(record.Key), record.StoredAt, resolution.Descriptor, - document) + decoded.Document) { StoredSchemaId = record.SchemaId, - StoredSchemaReadWhole = readWhole, + StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0 && unread == null, + UnreadSlots = unread, StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; @@ -4041,22 +4083,37 @@ namespace GameCult.Caching // catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file. internal void RefuseUndeclared(string key, CultDocumentDescriptor? written) { - var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); - if (foreign != null) - throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); - if (Entries.TryGetValue(key, out var held) && !HeldOwned(held, written)) - throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); + if (Refusal(key, written) is { } refusal) throw refusal; } + // The refusal of a write (of the written type) or a removal (written null) of the key, or null when this store's view lets it + // through. The one answer behind RefuseUndeclared and CultCache.IsWritable, so the query and the write cannot disagree. + internal CultSchemaConflictException? Refusal(string key, CultDocumentDescriptor? written) + { + var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + if (foreign != null) + return Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); + if (!Entries.TryGetValue(key, out var held) || HeldOwned(held, written)) + return null; + return held.UnreadSlots is { Explained: false } unread + ? UnreadSlotsRefusal(held, key, unread) + : Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); + } + + // The one ownership predicate of a held record. A mark nothing explains (anything but a depth-0 slot the catalog comparison + // reported ignored) keeps the record unowned whatever is declared: declaring the reader's own id consents to nothing. private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) => written == null - ? Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole + ? !Unexplained(held) && (Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole) : UpsertOwned(held, written); + private static bool Unexplained(CultStoredDocument held) => held.UnreadSlots is { Explained: false }; + // Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held // record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed // cache's staging and a store-less cache's. internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) => + !Unexplained(held) && CultDocumentRegistry.WrittenOwns( written.ToCatalogEntry(), held.StoredSchemaId, @@ -4358,6 +4415,16 @@ namespace GameCult.Caching refused[0].Key, inner); + // The refusal of a write or removal of a record held read-only for the slots its payload carries past this build's. It names the + // type and the counts, never a slot value. + private CultSchemaConflictException UnreadSlotsRefusal(CultStoredDocument held, string key, CultUnreadSlots unread) => new( + $"{held.Descriptor.SchemaName} record {key} is read-only: {unread.TypeName} was read with {unread.SlotCount} slots where this build declares {unread.DeclaredCount} " + + $"({unread.Occurrences} object(s) in the record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.", + held.StoredSchemaId, + new[] { held.Descriptor.SchemaName }, + key, + unread); + // A flush or commit succeeded: nothing staged is left, writes or checks. protected void MarkFlushSucceeded() { @@ -4461,6 +4528,9 @@ namespace GameCult.Caching protected abstract byte[] SerializePayload(object document, Type documentType); protected abstract object DeserializePayload(Type documentType, byte[] payload); + // A decode that also notes the slots it read past. A store kind that does not override it never marks. + internal virtual CultDecoded DeserializeMarked(Type documentType, byte[] payload) => new(DeserializePayload(documentType, payload), null); + public override string ToString() => FileInfo.FullName; public override void PullAll() => Held(PullAllCore); @@ -4495,7 +4565,7 @@ namespace GameCult.Caching continue; } - var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report); + var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializeMarked, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; } @@ -4963,7 +5033,7 @@ namespace GameCult.Caching var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)); if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null) return null; - return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _); + return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializeMarked, out _); } // The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes: diff --git a/src/GameCult.Caching/CultSchemaConflictException.cs b/src/GameCult.Caching/CultSchemaConflictException.cs index bbaff988..376b4c08 100644 --- a/src/GameCult.Caching/CultSchemaConflictException.cs +++ b/src/GameCult.Caching/CultSchemaConflictException.cs @@ -5,7 +5,8 @@ namespace GameCult.Caching /// /// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the /// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a - /// . Nothing is written, so the store stays as it was. It names the id, the schema names involved + /// , or a write or removal of a record held read-only because its payload carries slots past the + /// reader's declared slots ( is set). Nothing is written, so the store stays as it was. It names the id, the schema names involved /// and a record key. /// public sealed class CultSchemaConflictException : InvalidOperationException @@ -27,8 +28,18 @@ namespace GameCult.Caching RecordKey = recordKey; } + /// The refusal of a record held read-only for the slots it carries past this build's: the same refusal, with the finding. + public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey, CultUnreadSlots unreadSlots) + : this(message, schemaId, schemaNames, recordKey) + { + UnreadSlots = unreadSlots; + } + public string SchemaId { get; } public string[] SchemaNames { get; } public string RecordKey { get; } + + /// Set when the refusal is for unread slots; null for every other refusal. + public CultUnreadSlots? UnreadSlots { get; } } } diff --git a/src/GameCult.Caching/CultUnreadSlots.cs b/src/GameCult.Caching/CultUnreadSlots.cs new file mode 100644 index 00000000..f6d0416a --- /dev/null +++ b/src/GameCult.Caching/CultUnreadSlots.cs @@ -0,0 +1,53 @@ +namespace GameCult.Caching +{ + /// + /// What a store load found in a record's payload past the slots this build declares, at any depth: the first type read with extra + /// slots and how many slots it carried against how many it declares, and how many objects in the record did. It holds no slot + /// value, so it can be named in a refusal. A record carrying it is read-only () unless every + /// mark is explained. + /// + public sealed class CultUnreadSlots + { + internal CultUnreadSlots(string typeName, int slotCount, int declaredCount, int occurrences, bool allAtDepthZero, bool explained) + { + TypeName = typeName; + SlotCount = slotCount; + DeclaredCount = declaredCount; + Occurrences = occurrences; + AllAtDepthZero = allAtDepthZero; + Explained = explained; + } + + /// The first type read with extra slots. + public string TypeName { get; } + + /// The slots that object carried. + public int SlotCount { get; } + + /// The slots this build declares for that type (highest key plus one). + public int DeclaredCount { get; } + + /// The objects in the record that carried extra slots. + public int Occurrences { get; } + + /// + /// True only when every mark is at depth 0 and the schema comparison reported ignored slots under the id the record was stored + /// under: the loss a compatible-id declaration consents to (cultlib-gaps:ruling:slot-mark-consent). Any other mark leaves the + /// record read-only whatever is declared. + /// + public bool Explained { get; } + + // Every mark the decode noted sat in the record's own object, the half of "explained" the payload decides. + internal bool AllAtDepthZero { get; } + + internal CultUnreadSlots Explaining(bool explained) => + new(TypeName, SlotCount, DeclaredCount, Occurrences, AllAtDepthZero, explained); + } + + // A payload decoded by a store kind, with the extra slots the decode noted. A store kind that does not note them reports none. + internal readonly struct CultDecoded(object document, CultUnreadSlots? unreadSlots) + { + public object Document { get; } = document; + public CultUnreadSlots? UnreadSlots { get; } = unreadSlots; + } +} diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs new file mode 100644 index 00000000..03630c3a --- /dev/null +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -0,0 +1,310 @@ +#nullable enable +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using Surface = GameCult.Caching.Tests.UnionParityVectorTests.UnionSurface; + +namespace GameCult.Caching.Tests +{ + // cultlib-gaps:ruling:extra-trailing-slots and :slot-mark-consent. A record whose payload carries a slot past its reader's declared + // slots, at any depth, is held read-only: every staged write or removal of it is refused before anything is staged. A compatible-id + // declaration consents only to depth-0 slots the catalog comparison reports ignored under a listed compatible id. + public class SlotReadOnlyTests + { + private const string Extra = "extra-slot"; + + // Same members as UnionSurface under a new version, listing OldId compatible: the declaration that must not consent to a nested mark. + private const string OldId = "tests.slot_readonly.old"; + + [CultDocument("tests.union_surface", "tests.union_surface.v2", CompatibleSchemaIds = new[] { OldId })] + [MessagePackObject] + public sealed class SurfaceDeclaring + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public int Version; + [Key(2)] public UnionParityVectorTests.Node? Root; + [Key(3)] public List Outcomes = new(); + [Key(4)] public UnionParityVectorTests.Item? Part; + [Key(5)] public UnionParityVectorTests.CraftedItem? Crafted; + [Key(6)] public UnionParityVectorTests.Node? Missing; + } + + [CultDocument("tests.slot_unit", "tests.slot_unit.v1")] + [MessagePackObject] + public sealed class UnitDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public UnionParityVectorTests.UnitOutcome? Outcome; + } + + // Slot 1 and 2 are retired gaps. + [CultDocument("tests.slot_gap", "tests.slot_gap.v1")] + [MessagePackObject] + public sealed class GapDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(3)] public string Tail = string.Empty; + } + + private static readonly CultDocumentRegistry SurfaceRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(Surface) }); + private static readonly CultDocumentRegistry DeclaringRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(SurfaceDeclaring) }); + private static readonly CultDocumentRegistry UnitRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(UnitDoc) }); + private static readonly CultDocumentRegistry GapRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(GapDoc) }); + private static readonly CultRecordKey Marked = new("marked"); + private static readonly CultRecordKey Clean = new("clean"); + private static readonly CultRecordKey Unheld = new("unheld"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-slotro-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) Directory.Delete(_directory, recursive: true); + } + + private static CultCache Open(string path, CultDocumentRegistry registry) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + + private static byte[] Emit(WriterAction action) + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + action(ref writer); + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + private delegate void WriterAction(ref MessagePackWriter writer); + + // A store of two records of the given type, the marked one carrying the payload and the clean one an ordinary value. + private string Seed(CultDocumentRegistry registry, Type type, Func clean, byte[] markedPayload, string name = "slots.cc") + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, registry)) + { + cache.UpsertAsync(type, clean(), Marked).GetAwaiter().GetResult(); + cache.UpsertAsync(type, clean(), Clean).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.Records.Single(record => record.Key == Marked.Value).Payload = markedPayload; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + private string SeedSurface(string vector) => + Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, UnionParityVectorTests.Committed(vector)); + + private static CultStoredDocument Stored(CultCache cache, CultRecordKey key) => cache.GetStored(key)!; + + // ---- a mark at every depth, with the type and both counts ---- + + [TestCase("extra-slot-record.msgpack", "UnionSurface", 8, 7, 1)] + [TestCase("extra-slot-nested-record.msgpack", nameof(UnionParityVectorTests.Composition), 3, 2, 1)] + [TestCase("extra-slot-arm.msgpack", nameof(UnionParityVectorTests.Box), 3, 2, 1)] + [TestCase("extra-slot-list-element.msgpack", nameof(UnionParityVectorTests.Text), 3, 2, 1)] + [TestCase("depth-skipped-at-limit.msgpack", "UnionSurface", 8, 7, 1)] + public void AMarkAtAnyDepthHoldsTheRecordReadOnly(string vector, string typeName, int slots, int declared, int occurrences) + { + var path = SeedSurface(vector); + using var cache = Open(path, SurfaceRegistry); + var unread = Stored(cache, Marked).UnreadSlots; + Assert.That(unread, Is.Not.Null, vector); + Assert.That(unread!.TypeName, Is.EqualTo(typeName), vector); + Assert.That(unread.SlotCount, Is.EqualTo(slots), vector); + Assert.That(unread.DeclaredCount, Is.EqualTo(declared), vector); + Assert.That(unread.Occurrences, Is.EqualTo(occurrences), vector); + Assert.That(unread.Explained, Is.False, "the reader's own id is not a compatible-id declaration: nothing explains the mark"); + Assert.That(cache.IsWritable(Marked), Is.False, vector); + Assert.That(cache.Get(Marked), Is.Not.Null, "a read-only record is still served"); + Assert.That(Stored(cache, Clean).UnreadSlots, Is.Null); + Assert.That(cache.IsWritable(Clean), Is.True); + Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable"); + } + + [Test] + public void AUnitArmWithAnExtraSlotHoldsTheRecordReadOnly() + { + var arm = UnionParityVectorTests.Committed("extra-slot-unit-arm.msgpack"); + var payload = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("u"); w.WriteRaw(arm); }); + var path = Seed(UnitRegistry, typeof(UnitDoc), () => new UnitDoc { Name = "u" }, payload); + using var cache = Open(path, UnitRegistry); + var unread = Stored(cache, Marked).UnreadSlots; + Assert.That(unread, Is.Not.Null); + Assert.That(unread!.TypeName, Is.EqualTo(nameof(UnionParityVectorTests.Denied2))); + Assert.That(unread.SlotCount, Is.EqualTo(1)); + Assert.That(unread.DeclaredCount, Is.EqualTo(0)); + Assert.That(cache.IsWritable(Marked), Is.False); + Assert.That(cache.IsWritable(Clean), Is.True); + } + + // ---- every staged write and removal of a marked key is refused, nothing else is ---- + + private static void AssertSlotRefusal(TestDelegate act, string what, string type, int slots, int declared) + { + var refusal = Assert.Throws(act, what)!; + Assert.That(refusal.UnreadSlots, Is.Not.Null, what); + Assert.That(refusal.RecordKey, Is.EqualTo("marked"), what); + Assert.That(refusal.Message, Does.Contain("is read-only: " + type + " was read with " + slots + " slots where this build declares " + declared) + .And.Contain("record marked"), what); + Assert.That(refusal.Message, Does.Not.Contain(Extra), $"{what}: the refusal never echoes a slot value"); + } + + [Test] + public void EveryWriteAndRemovalOfAMarkedRecordIsRefusedAndAnotherKeyStillWrites() + { + var path = SeedSurface("extra-slot-nested-record.msgpack"); + var before = File.ReadAllBytes(path); + using (var cache = Open(path, SurfaceRegistry)) + { + var composition = nameof(UnionParityVectorTests.Composition); + AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle(Marked))), "upsert", composition, 3, 2); + AssertSlotRefusal(() => cache.Commit(batch => batch.UpsertVariant(Marked, Clean)), "variant", composition, 3, 2); + AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "remove", composition, 3, 2); + AssertSlotRefusal(() => cache.Commit(batch => { batch.Upsert(new Surface { Id = "other" }, new CultRecordHandle(Clean)); batch.Remove(Marked); }), "mixed batch", composition, 3, 2); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused writes leave the store as it was"); + Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle(Clean))), Is.True); + } + + Assert.That(CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Count, Is.EqualTo(2)); + } + + // ---- consent ---- + + [Test] + public void ADeclaredIdDoesNotConsentToAnUnexplainedMark() + { + // The reader's own id is declared; the mark at depth 0 is in no compatible id's comparison: still read-only. + var path = SeedSurface("extra-slot-record.msgpack"); + using var cache = Open(path, SurfaceRegistry); + AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle(Marked))), "depth 0 under the reader's own id", "UnionSurface", 8, 7); + AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "removal", "UnionSurface", 8, 7); + } + + [Test] + public void AListedCompatibleIdDoesNotConsentToANestedMark() + { + // The record's stored id is one the reader lists compatible; the extra slot is nested, so the declaration does not reach it. + var path = SeedSurface("extra-slot-nested-record.msgpack"); + Relabel(path, OldId); + using var cache = Open(path, DeclaringRegistry); + Assert.That(Stored(cache, Marked).StoredSchemaId, Is.EqualTo(OldId)); + Assert.That(cache.IsWritable(Marked), Is.False); + AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Marked))), "nested under a listed id", nameof(UnionParityVectorTests.Composition), 3, 2); + Assert.That(cache.IsWritable(Clean), Is.True, "the clean record under the listed id is consented to (declared)"); + Assert.That(cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Clean))), Is.True); + } + + private static void Relabel(string path, string id) + { + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.SchemaCatalog.Single().SchemaId = id; + foreach (var record in snapshot.Records) record.SchemaId = id; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + + // ---- carried verbatim ---- + + [Test] + public void AMarkedRecordIsCarriedByteForByteWhenAnotherRecordIsWritten() + { + var path = Path.Combine(_directory, "extra-slot-store.cc"); + File.WriteAllBytes(path, UnionParityVectorTests.Committed("extra-slot-store.cc")); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var original = before.Records.Single(record => record.Key == Marked.Value); + using (var cache = Open(path, SurfaceRegistry)) + { + Assert.That(cache.IsWritable(Marked), Is.False); + Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle(Clean))), Is.True); + } + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var carried = after.Records.Single(record => record.Key == Marked.Value); + Assert.That(carried.Payload, Is.EqualTo(original.Payload)); + Assert.That(carried.SchemaId, Is.EqualTo(original.SchemaId)); + Assert.That(carried.StoredAt, Is.EqualTo(original.StoredAt)); + Assert.That(after.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString), + Is.EqualTo(before.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString))); + Assert.That(after.Records.Single(record => record.Key == Clean.Value).Payload, Is.Not.EqualTo(before.Records.Single(record => record.Key == Clean.Value).Payload)); + } + + // ---- slots short of the reader's, and gaps below the highest key, do not mark ---- + + [Test] + public void MissingTrailingSlotsDefaultAndAGapSlotDoesNotMark() + { + var short1 = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(1); w.Write("short"); }); + var path = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, short1); + using (var cache = Open(path, GapRegistry)) + { + Assert.That(cache.Get(Marked)!.Tail, Is.EqualTo(string.Empty), "a missing trailing slot keeps its default"); + Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null); + Assert.That(cache.IsWritable(Marked), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(new GapDoc { Name = "short", Tail = "t" }, new CultRecordHandle(Marked))), Is.True); + } + + var gapped = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(4); w.Write("gap"); w.WriteNil(); w.Write("in a retired slot"); w.Write("tail"); }); + var gapPath = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, gapped, "gap.cc"); + using (var cache = Open(gapPath, GapRegistry)) + { + Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, "a non-nil value in a retired gap below the highest key does not mark"); + Assert.That(cache.IsWritable(Marked), Is.True); + } + } + + // ---- the catalog's consent still works, and is reported as explained ---- + + [CultDocument("tests.slot_wide", "tests.slot_wide.v1")] + [MessagePackObject] + public sealed class WideDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public string Colour = string.Empty; + [Key(2)] public string Extra = string.Empty; + } + + [CultDocument("tests.slot_wide", "tests.slot_wide.v2")] + [MessagePackObject] + public sealed class NarrowDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public string Colour = string.Empty; + } + + [Test] + public void ADepthZeroSlotTheCatalogReportsIsExplainedAndTheUndeclaredRecordStaysUnwritable([Values] bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + CacheBackingStore Store() => directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path); + using (var writer = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) }))) + { + writer.AddBackingStore(Store()); + Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "w", Extra = "kept?" }, new CultRecordHandle(Marked))), Is.True); + } + + using var reader = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) })); + reader.AddBackingStore(Store()); + var unread = Stored(reader, Marked).UnreadSlots; + Assert.That(unread, Is.Not.Null, "the store kind decodes through the marking path"); + Assert.That(unread!.Explained, Is.True, "depth 0 and reported by the catalog comparison"); + Assert.That(unread.TypeName, Is.EqualTo(nameof(NarrowDoc))); + Assert.That(unread.SlotCount, Is.EqualTo(3)); + Assert.That(unread.DeclaredCount, Is.EqualTo(2)); + Assert.That(reader.IsWritable(Marked), Is.False, "the wide id is undeclared: unowned, though the mark is explained"); + Assert.Throws(() => reader.Commit(batch => batch.Remove(Marked))); + } + } +} diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 9418a680..08a37243 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -58,7 +58,7 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}"); } - private static byte[] Committed(string name) + internal static byte[] Committed(string name) { var path = VectorPath(name); Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); @@ -297,6 +297,9 @@ namespace GameCult.Caching.Tests Surface(extra: new Extra(RootArm: true))), new Case("extra-slot-list-element.msgpack", "read-only", "Node", 1, "list-element", Surface(extra: new Extra(ListElement: true))), + // A bare UnitOutcome arm (the fixture's unit-arm union), not a UnionSurface payload: [2, [extra]] where Denied2 declares no slot. + new Case("extra-slot-unit-arm.msgpack", "read-only", "UnitOutcome", 2, "unit-arm", + Raw(ArmOf(2, ExtraSlot))), }; private static byte[] OutcomesFile() => Raw((ref MessagePackWriter w) => @@ -317,6 +320,7 @@ namespace GameCult.Caching.Tests "Node" => typeof(Node).FullName!, "Outcome" => typeof(Outcome).FullName!, "CraftedItem" => typeof(CraftedItem).FullName!, + "UnitOutcome" => typeof(UnitOutcome).FullName!, "UnionSurface" => typeof(UnionSurface).FullName!, _ => throw new ArgumentOutOfRangeException(nameof(logical), logical, null) }; @@ -413,6 +417,34 @@ namespace GameCult.Caching.Tests } } + // The store the slot-readonly tests open and the other runtimes read: the extra-slot-nested-record payload as the record "marked" + // beside the ordinary value as "clean". The store's clock is pinned as union-surface.cc's. + [Test] + public void ExtraSlotStoreVectorHoldsAMarkedAndACleanRecord() + { + var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}"); + Directory.CreateDirectory(work); + try + { + var scratch = Path.Combine(work, "extra-slot-store.cc"); + using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry, StoreFlushOnDispose = true })) + { + cache.Clock = () => StoredAt; + cache.UpsertAsync(typeof(UnionSurface), Constructed(), new CultRecordKey("marked")).GetAwaiter().GetResult(); + cache.UpsertAsync(typeof(UnionSurface), Constructed(), new CultRecordKey("clean")).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(scratch)); + snapshot.Records.Single(record => record.Key == "marked").Payload = Committed("extra-slot-nested-record.msgpack"); + Check("extra-slot-store.cc", CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + finally + { + Directory.Delete(work, recursive: true); + } + } + // The value names every arm shape the vectors exist to hold other runtimes to. [Test] public void ConstructedValueExercisesEveryArmShape() @@ -475,8 +507,11 @@ namespace GameCult.Caching.Tests else { Assert.That(outcome, Is.EqualTo("read-only"), file); - // The record is read whole; what makes it read-only is the slot rule of the slot-tolerance cut, not asserted here. - Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); + // The extra slot is read past; that the record then loads read-only is asserted in SlotReadOnlyTests, store by store. + if (union == "UnitOutcome") + Assert.That(MessagePackSerializer.Deserialize(payload, UnitOptions), Is.TypeOf(), $"{file}: the extra slot is read past"); + else + Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); } } } diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 17f9ba26..020490fe 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -1039,7 +1039,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndStaysStoredUntilTheVariantIsWritten() + public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndHoldsTheVariantReadOnly() { var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", CultVariantOverride.Set(99, new byte[] { 0x01 }), @@ -1059,14 +1059,22 @@ namespace GameCult.Caching.Tests { Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 })); Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); - reopened.Commit(batch => batch.UpsertVariant(new CultRecordKey("hand-variant"), new CultRecordKey("hand-base"), new[] + var variant = new CultRecordKey("hand-variant"); + Assert.That(reopened.IsWritable(variant), Is.False, "an override of a slot the type lacks, which the catalog did not report, is a mark nothing explains"); + var overrides = new[] { reopened.Override(nameof(VariantGear.Name), "hand variant"), reopened.Override(nameof(VariantGear.Code), "hand-code") - })); + }; + var rewrite = Assert.Throws(() => reopened.Commit(batch => batch.UpsertVariant(variant, new CultRecordKey("hand-base"), overrides)))!; + Assert.That(rewrite.UnreadSlots, Is.Not.Null); + Assert.That(rewrite.UnreadSlots!.TypeName, Is.EqualTo(nameof(VariantGear))); + Assert.That(rewrite.UnreadSlots.SlotCount, Is.EqualTo(100)); + Assert.Throws(() => reopened.Commit(batch => batch.Flatten(variant))); + Assert.Throws(() => reopened.Commit(batch => batch.Remove(variant))); } - Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "a write of the variant stores only what it was given"); + Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2, 99 }), "the refused writes left the variant as stored"); } private static int[] OverriddenSlotsOn(string path, string key) => diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index c6516fbb..e55cd17f 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -58,9 +58,19 @@ of it refused). A reader decodes the value of `union-surface.msgpack` from each; | `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` | | `extra-slot-arm.msgpack` | arm | the Root Box arm | | `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots | +| `extra-slot-unit-arm.msgpack` | unit-arm | not a UnionSurface: a bare UnitOutcome arm `[2, [extra]]` where Denied2 declares no slot (union `UnitOutcome`, key 2) | -The C# reference reads these today and its slot-tolerance cut is what makes them read-only; this cut asserts only -that the value is read. +`depth-skipped-at-limit.msgpack` (bounds file) is also `read-only`: its extra slot is a deeply nested value at depth 0. +Five rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm, list-element, unit-arm). + +`extra-slot-store.cc` is a single-file store holding two records of `tests.union_surface`: `marked` (the +`extra-slot-nested-record.msgpack` payload) and `clean` (the ordinary value); the store's clock is pinned to +2026-01-01T00:00:00Z as `union-surface.cc`'s. + +Every runtime asserts the same for a `read-only` payload loaded as a store record: the value is read past the extra +slot, the record reports the unread slots (the first type read with them, the slots it carried and the slots the +reader declares) and is not writable, and a write or removal of it is refused before anything is staged. Opening +`extra-slot-store.cc` and writing `clean` leaves `marked` byte for byte, with its schema id and its catalog entry. ## Bounds (`bounds-outcomes.msgpack`) diff --git a/tests/vectors/union-parity/extra-slot-store.cc b/tests/vectors/union-parity/extra-slot-store.cc new file mode 100644 index 0000000000000000000000000000000000000000..5d9c77417de59f473f99f3aafcbb72e497a92acf GIT binary patch literal 2809 zcmd5-OHbQC5a!ulxc1)W3K)AgcH*e2RONsowX_1trK+m+?ij54f$R+>s`wAZL4so+ zfm9W}v{KIzRewbKYdVgNaS)djh?-bRc6J`$%zQI$f9FkCa>lnniDVFhvP1U|`^(W5 zBf3!olNq`JdYM>i6;Q3LS*C54MV0FswK#AgES0H1bVjra6{cyaTE!NGajsM2hlEH* zO4D}&|8>+1ZN}l}6*6T<%VSy^<@nXhf?66gh;0&GR4huNYS~n$6`&Q{0Ah)%!8HSw z2xw}xY7%Cul(IjwbMHzKCgiad=0T}ex*}j{Aye6aFhX!wYauLD^uV)Ft5kc5e%(Np zYU)DCPhB?zD#+G$5@xLxN$`|489M$Zik)Z`YzV>Ucxy8^0`$fFR(_By+yuE+fa*70 z7Y%&@yN_LmVa!YtrK!lU3406YFti-Tc@~$kG<-?P?LZkMbb*EBSXKipQ4VAo>r_XI zDRtx+)&U(LkN0Z%b0i9--gS~@yXT77TPO=dC zWc0eE_S0X^C$ztDJCm8lndu}WGb+x6ajE@mHzQ%+94*xXcT-Pyx*4fP1(T3z;37Ua z<}NV*##-NhKvZHZsv17)s#+U7ikg1r2dCeY_YS++D}Fkjh)=Uy{NZXlo*WWr$6MgK zK|Iwis*$*Zt9X0ZSg7%^vCuy5#m+aO-mqIB?}wdm znmMt0AKIwfdCMGG2*NakaknTS@zj3tOi|%vi+LV|bNIKj2`rrH9h`V9d;?;g? -- 2.47.3 From 53d2f117800de3b582f7668d9867d7025b07a30d Mon Sep 17 00:00:00 2001 From: soul Date: Tue, 10 Nov 2026 14:35:10 +0000 Subject: [PATCH 02/20] Contracts: extra slots are read-only, not ignored Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 11 ++++++++--- .../Contracts/cultcache-schema-compatibility.md | 13 ++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 93faf63f..851d261e 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -202,6 +202,12 @@ flatten, remove, commit) is refused with the typed schema conflict before anythi Declaring its id compatible on any registered type makes it writable and is the consent to the loss: the next write stores it under the writing type's own id and sheds the slots. +A record whose payload carries a slot past the slots its reader declares, at any depth (nested object, list +element, union arm, unit arm), is read-only the same way, whatever id it is stored under. The one consent +is the declaration above, and it reaches only depth-0 slots the schema comparison reports ignored under a +listed compatible id: a mark anywhere else, or at depth 0 under the reader's own id, is not consented to. +`CultCache.IsWritable(key)` answers the same question the staging refusal asks. + The rule is checked again where the write lands, under the store's lock: a write or removal at a key whose durable record is not owned, because another writer rewrote it after this cache loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can @@ -535,15 +541,14 @@ Read directly when: Allowed when: - persisted schema has missing slots that the local schema can default -- persisted schema has extra slots the local reader can safely ignore or preserve - slot meanings remain compatible -Soft migration should emit warnings, not silent shrugs. +A persisted schema with extra slots the local reader lacks is not soft: the record is read-only (see the ownership +rule above). Soft migration should emit warnings, not silent shrugs. Typical warnings: - missing slot defaulted -- unknown extra slot ignored - nullable/default behavior widened - field preserved but not surfaced natively diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index dc7e7bea..c638050e 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -107,10 +107,21 @@ persisted slots honestly: - persisted schema id differs, but the embedded catalog points to a compatible local schema id -- persisted slot is missing locally and can be ignored +- persisted slot is missing locally and the record is read-only until its id is declared compatible + (see Extra slots are read-only, below) - local slot is missing in persisted data and can fall back to the local default value +Extra slots are read-only. A record whose payload carries a slot past the slots its reader declares, at any +depth (the record, a nested object, a list element, a union arm, a unit arm), is loaded read-only: it is +served, `CultStoredDocument.UnreadSlots` names the first type read with extra slots and both counts, and +`CultCache.IsWritable(key)` is false. A staged write or removal of it is refused with +`CultSchemaConflictException` (`UnreadSlots` set; the message names the type and the counts, never a slot +value). A missing trailing slot keeps its default and stays writable; a non-nil value in a retired gap below +the highest key is not extra. Declaring an id compatible consents only to depth-0 slots the schema comparison +reports ignored under that id; any other mark (nested, list element, arm, unit arm, or depth 0 under the +reader's own id) keeps the record read-only whatever is declared. + The cache emits a typed migration report with: - exact vs compatible-drift classification -- 2.47.3 From 4f1d5e9d23e6df8cf55bb359c2af8c4add8435fa Mon Sep 17 00:00:00 2001 From: soul Date: Tue, 10 Nov 2026 14:40:28 +0000 Subject: [PATCH 03/20] SlotReadOnlyTests: a compatible id consents only to the depth-0 slot the catalog reports Co-Authored-By: Claude Sonnet 5.5 --- .../SlotReadOnlyTests.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index 03630c3a..040d9c97 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -208,6 +208,49 @@ namespace GameCult.Caching.Tests Assert.That(cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Clean))), Is.True); } + // The record stored under a listed compatible id whose catalog entry has one more member than the reader (slot 7), the payload + // carrying that slot at depth 0: the loss a declaration consents to. With a nested mark beside it, nothing is consented to. + [TestCase("union-surface.msgpack", true, TestName = "ADepthZeroSlotTheCatalogReportsUnderAListedIdIsConsentedTo")] + [TestCase("extra-slot-nested-record.msgpack", false, TestName = "ANestedMarkBesideACatalogReportedSlotIsNotConsentedTo")] + public void ACompatibleIdConsentsOnlyToTheDepthZeroSlotTheCatalogReports(string vector, bool consented) + { + var original = UnionParityVectorTests.Committed(vector); + var path = Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, AppendRootSlot(original, "wider writer")); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + var last = entry.Members.OrderBy(member => member.Slot).Last(); + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 7, MemberName = "Wider", TypeName = last.TypeName }).ToArray(); + entry.SchemaId = OldId; + foreach (var record in snapshot.Records) record.SchemaId = OldId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using var cache = Open(path, DeclaringRegistry); + var unread = Stored(cache, Marked).UnreadSlots; + Assert.That(unread, Is.Not.Null); + Assert.That(unread!.Explained, Is.EqualTo(consented)); + Assert.That(cache.IsWritable(Marked), Is.EqualTo(consented)); + if (consented) + Assert.That(cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Marked))), Is.True); + else + Assert.Throws(() => cache.Commit(batch => batch.Remove(Marked))); + } + + // The payload's root array with one string slot appended. + private static byte[] AppendRootSlot(byte[] payload, string value) + { + var reader = new MessagePackReader(payload); + var count = reader.ReadArrayHeader(); + var start = (int)reader.Consumed; + for (var i = 0; i < count; i++) reader.Skip(); + var end = (int)reader.Consumed; + return Emit((ref MessagePackWriter w) => + { + w.WriteArrayHeader(count + 1); + w.WriteRaw(payload.AsSpan(start, end - start)); + w.Write(value); + }); + } + private static void Relabel(string path, string id) { var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); -- 2.47.3 From e09700d439c8d8a22af53989b0bc03fec02246d4 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 23:58:11 +0000 Subject: [PATCH 04/20] Slot tests read vectors through VectorFiles; the variant read-past test is named for what it pins SlotReadOnlyTests reads the union-parity vectors through VectorFiles.Committed, the shared reader, so UnionParityVectorTests.Committed stays private. The variant test that only pins the read no longer says the extra slot is ignored: under the slot rule it is read past and the record is read-only. Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs | 8 ++++---- tests/GameCult.Caching.Tests/UnionParityVectorTests.cs | 2 +- tests/GameCult.Caching.Tests/VariantTests.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index 040d9c97..f0caf141 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -106,7 +106,7 @@ namespace GameCult.Caching.Tests } private string SeedSurface(string vector) => - Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, UnionParityVectorTests.Committed(vector)); + Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, VectorFiles.Committed("union-parity", vector)); private static CultStoredDocument Stored(CultCache cache, CultRecordKey key) => cache.GetStored(key)!; @@ -138,7 +138,7 @@ namespace GameCult.Caching.Tests [Test] public void AUnitArmWithAnExtraSlotHoldsTheRecordReadOnly() { - var arm = UnionParityVectorTests.Committed("extra-slot-unit-arm.msgpack"); + var arm = VectorFiles.Committed("union-parity", "extra-slot-unit-arm.msgpack"); var payload = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("u"); w.WriteRaw(arm); }); var path = Seed(UnitRegistry, typeof(UnitDoc), () => new UnitDoc { Name = "u" }, payload); using var cache = Open(path, UnitRegistry); @@ -214,7 +214,7 @@ namespace GameCult.Caching.Tests [TestCase("extra-slot-nested-record.msgpack", false, TestName = "ANestedMarkBesideACatalogReportedSlotIsNotConsentedTo")] public void ACompatibleIdConsentsOnlyToTheDepthZeroSlotTheCatalogReports(string vector, bool consented) { - var original = UnionParityVectorTests.Committed(vector); + var original = VectorFiles.Committed("union-parity", vector); var path = Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, AppendRootSlot(original, "wider writer")); var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); var entry = snapshot.SchemaCatalog.Single(); @@ -265,7 +265,7 @@ namespace GameCult.Caching.Tests public void AMarkedRecordIsCarriedByteForByteWhenAnotherRecordIsWritten() { var path = Path.Combine(_directory, "extra-slot-store.cc"); - File.WriteAllBytes(path, UnionParityVectorTests.Committed("extra-slot-store.cc")); + File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-store.cc")); var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); var original = before.Records.Single(record => record.Key == Marked.Value); using (var cache = Open(path, SurfaceRegistry)) diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 08a37243..22ef82c9 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -58,7 +58,7 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}"); } - internal static byte[] Committed(string name) + private static byte[] Committed(string name) { var path = VectorPath(name); Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 020490fe..cdbdcf58 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -513,7 +513,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AnOverrideOfASlotTheTypeNeverHadIsIgnoredOnLoadAsAPlainRecordIgnoresAnExtraSlot() + public void AnOverrideOfASlotTheTypeNeverHadIsReadPastOnLoadAsAPlainRecordReadsPastAnExtraSlot() { var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", CultVariantOverride.Set(99, new byte[] { 0x01 }), -- 2.47.3 From 4e9820de6fdbe5fdfc2834b293b38dacfbc6c804 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 23:58:11 +0000 Subject: [PATCH 05/20] union-parity README: what every runtime walks for a read-only payload Per vector: the object carrying the extra slot, its depth and the mark (type, slots carried, slots declared); the counting rule (highest key + 1, short arrays and retired gaps do not mark, a union adds no depth); the store-record assertions, the consent rule and the extra-slot-store.cc carry check that cut-slot-readonly-ts/-py/-rs assert. Co-Authored-By: Claude Opus 5.5 --- tests/vectors/union-parity/README.md | 63 +++++++++++++++++++--------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index e55cd17f..618562f1 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -48,29 +48,54 @@ with one part replaced. ## Extra trailing slot (outcome `read-only`) -Each payload is `union-surface.msgpack` with one slot appended past the reader's last declared slot of one -record (ruling `cultlib-gaps:ruling:extra-trailing-slots`: the record is read, read-only at any depth, every write -of it refused). A reader decodes the value of `union-surface.msgpack` from each; `detail` says where the slot is. +Rulings `cultlib-gaps:ruling:extra-trailing-slots` and `cultlib-gaps:ruling:slot-mark-consent`: a payload carrying a +slot past its reader's declared slots, in any object at any depth, is read, held read-only, and every write or +removal of it is refused before anything is staged. The first four payloads are `union-surface.msgpack` with the +string `"extra-slot"` appended as one more slot of one object; a reader decodes the value of `union-surface.msgpack` +from each. The fifth is a bare arm of union `UnitOutcome` (0 Waiting `[Message]`; 2 Denied2, -2147483648 Floor2 and +2147483647 Ceiling2, each `[]`), the union of the `unit-arm-*` files; a reader decodes Denied2 from it. -| File | detail | Record with the extra slot | -|---|---|---| -| `extra-slot-record.msgpack` | record | the document (slot 7) | -| `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` | -| `extra-slot-arm.msgpack` | arm | the Root Box arm | -| `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots | -| `extra-slot-unit-arm.msgpack` | unit-arm | not a UnionSurface: a bare UnitOutcome arm `[2, [extra]]` where Denied2 declares no slot (union `UnitOutcome`, key 2) | +An object's declared slots are its highest key plus one. Only an array longer than that marks: a short array +(missing trailing slots) keeps the defaults and is writable, and a non-nil value in a retired slot below the highest +key (Image's slot 2) does not mark. A union adds no depth: an arm's array is counted against the arm's own slots. -`depth-skipped-at-limit.msgpack` (bounds file) is also `read-only`: its extra slot is a deeply nested value at depth 0. -Five rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm, list-element, unit-arm). +| File | detail | Object with the extra slot | Depth | Mark: type, slots carried, slots declared | +|---|---|---|---|---| +| `extra-slot-record.msgpack` | record | the document (slot 7) | 0 | UnionSurface, 8, 7 | +| `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` | nested | Composition, 3, 2 | +| `extra-slot-arm.msgpack` | arm | the Root Box arm | nested | Box, 3, 2 | +| `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots | nested | Text, 3, 2 | +| `extra-slot-unit-arm.msgpack` | unit-arm | Denied2 in `[2, ["extra-slot"]]` (union `UnitOutcome`, key 2) | as placed | Denied2, 1, 0 | +| `depth-skipped-at-limit.msgpack` (bounds file) | depth | the document (slot 7, a value of nested arrays) | 0 | UnionSurface, 8, 7 | -`extra-slot-store.cc` is a single-file store holding two records of `tests.union_surface`: `marked` (the -`extra-slot-nested-record.msgpack` payload) and `clean` (the ordinary value); the store's clock is pinned to -2026-01-01T00:00:00Z as `union-surface.cc`'s. +Each payload marks exactly one object. Five rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm, +list-element, unit-arm); the sixth `read-only` payload is in `bounds-outcomes.msgpack`. The payload nesting check +runs before the mark: `depth-skipped-past-limit.msgpack` is refused, never marked. -Every runtime asserts the same for a `read-only` payload loaded as a store record: the value is read past the extra -slot, the record reports the unread slots (the first type read with them, the slots it carried and the slots the -reader declares) and is not writable, and a write or removal of it is refused before anything is staged. Opening -`extra-slot-store.cc` and writing `clean` leaves `marked` byte for byte, with its schema id and its catalog entry. +### What a runtime asserts for a `read-only` payload + +Load it as a store record under the reader's own schema id (the C# reference puts each UnionSurface payload in a +`tests.union_surface` store; the unit arm goes in slot 1 of a document `[Name, Outcome: UnitOutcome]`). Then: + +1. the value is read past the extra slot; +2. the record reports the mark as in the table: the first type read with extra slots, the slots it carried and the + slots the reader declares, and the count of objects that carried extra slots (1 here); +3. the record is not writable (C#: `CultCache.IsWritable(key)` false), and a key no store holds is writable; +4. a replacing write, a variant write and a removal of the key (and a flatten, when the marked record is a variant) + are refused before anything is staged, + with the runtime's schema conflict refusal carrying the mark; its message names the type and both counts and + never contains `extra-slot`, the slot's value; +5. a write to another key of the same store succeeds. + +Consent: a mark is explained only when it is at depth 0 and the schema comparison of the record's stored id against +the reader reported ignored extra slots, i.e. the record is stored under an id the reader lists compatible and the +catalog entry for that id has more members. Declaring that id consents to an explained mark only. Every mark in the +table is unexplained (stored under the reader's own id), so each record stays read-only whatever is declared. + +`extra-slot-store.cc` is a single-file store of two `tests.union_surface` records, the store's clock pinned to +2026-01-01T00:00:00Z as `union-surface.cc`'s: `marked`, whose payload is `extra-slot-nested-record.msgpack`, and +`clean`, the value of `union-surface.msgpack`. Open it, assert `marked` read-only and `clean` writable, write `clean` +and flush: `marked`'s payload bytes, schema id and stored-at, and every catalog entry's bytes, equal the originals. ## Bounds (`bounds-outcomes.msgpack`) -- 2.47.3 From e692bd90d128e8fdb6c4ba79d59806d22178f237 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 23:58:11 +0000 Subject: [PATCH 06/20] CHANGELOG (org.gamecult.cultlib, Unreleased): extra slots are read-only (Breaking); IsWritable, CultUnreadSlots (Added) Behavioural break per docs/semver-policy.md (tightens the soft-drift classification; writes that succeeded now refuse). ApiCompat against cultlib-unity-v3.0.0 measures no signature break. Co-Authored-By: Claude Opus 5.5 --- unity/org.gamecult.cultlib/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 225bb3ae..c2fc2a1f 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -7,6 +7,12 @@ All notable changes to this package are documented in this file. ### Breaking - A single-file store holding two records under one key is unreadable: the reader refuses it with `CultStoreUnreadableException` naming the key (keys compare by ordinal code units, no case folding or normalisation), so a cache, `ReadStoreSnapshot` and a flush refuse the file and leave it as it is. Before, the later record silently replaced the earlier. A directory store reads its manifest through the same reader and refuses a duplicate key the same way. Repair the duplicate with the runtime that wrote it before reading the store here. +- A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. + +### Added + +- `CultCache.IsWritable(CultRecordKey)`: whether a write or removal of the key would be let through, answered by the same predicate the write uses (false for a foreign record, undeclared drift that drops slots, or unread slots; true for a key no store holds). Live and advisory: the write still decides. +- `CultUnreadSlots` (`TypeName`, `SlotCount`, `DeclaredCount`, `Occurrences`, `Explained`), `CultStoredDocument.UnreadSlots`, and `CultSchemaConflictException.UnreadSlots` with its constructor overload: what a load found past the reader's slots, holding no slot value. The refusal message names the type and both counts, never a value. ## [3.0.0] - 2026-11-11 -- 2.47.3 From 11520293af0ef5436ab3e23439cd456ab36874aa Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 00:30:02 +0000 Subject: [PATCH 07/20] Slot tests, red: written-count marks, flatten through an unowned base, consent capped, IsWritable is the write's answer Vectors: extra-slot-store.cc gains the unit-arm record (tests.slot_unit); extra-slot-consent.cc (records under tests.union_surface.wide) and slot-outcomes.msgpack are new. The consent reader (SurfaceDeclaring, listing the wide id compatible) and UnitDoc move into UnionParityVectorTests; the in-test consent store construction, Relabel and AppendRootSlot go. Red against 115bc0a2: 9 tests (spec cultlib-gaps:cut_spec:cut-slot-tolerance.r5). Co-Authored-By: Claude Opus 5.5 --- .../SlotReadOnlyTests.cs | 397 +++++++++++++----- .../UnionParityVectorTests.cs | 130 +++++- .../union-parity/extra-slot-consent.cc | Bin 0 -> 3479 bytes .../vectors/union-parity/extra-slot-store.cc | Bin 2809 -> 3716 bytes .../union-parity/slot-outcomes.msgpack | Bin 0 -> 322 bytes 5 files changed, 415 insertions(+), 112 deletions(-) create mode 100644 tests/vectors/union-parity/extra-slot-consent.cc create mode 100644 tests/vectors/union-parity/slot-outcomes.msgpack diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index f0caf141..daa4beba 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using GameCult.Caching.MessagePack; using MessagePack; +using MessagePack.Formatters; using NUnit.Framework; using Surface = GameCult.Caching.Tests.UnionParityVectorTests.UnionSurface; @@ -18,30 +19,6 @@ namespace GameCult.Caching.Tests { private const string Extra = "extra-slot"; - // Same members as UnionSurface under a new version, listing OldId compatible: the declaration that must not consent to a nested mark. - private const string OldId = "tests.slot_readonly.old"; - - [CultDocument("tests.union_surface", "tests.union_surface.v2", CompatibleSchemaIds = new[] { OldId })] - [MessagePackObject] - public sealed class SurfaceDeclaring - { - [Key(0)] [CultName] public string Id = string.Empty; - [Key(1)] public int Version; - [Key(2)] public UnionParityVectorTests.Node? Root; - [Key(3)] public List Outcomes = new(); - [Key(4)] public UnionParityVectorTests.Item? Part; - [Key(5)] public UnionParityVectorTests.CraftedItem? Crafted; - [Key(6)] public UnionParityVectorTests.Node? Missing; - } - - [CultDocument("tests.slot_unit", "tests.slot_unit.v1")] - [MessagePackObject] - public sealed class UnitDoc - { - [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] public UnionParityVectorTests.UnitOutcome? Outcome; - } - // Slot 1 and 2 are retired gaps. [CultDocument("tests.slot_gap", "tests.slot_gap.v1")] [MessagePackObject] @@ -52,8 +29,6 @@ namespace GameCult.Caching.Tests } private static readonly CultDocumentRegistry SurfaceRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(Surface) }); - private static readonly CultDocumentRegistry DeclaringRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(SurfaceDeclaring) }); - private static readonly CultDocumentRegistry UnitRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(UnitDoc) }); private static readonly CultDocumentRegistry GapRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(GapDoc) }); private static readonly CultRecordKey Marked = new("marked"); private static readonly CultRecordKey Clean = new("clean"); @@ -135,22 +110,6 @@ namespace GameCult.Caching.Tests Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable"); } - [Test] - public void AUnitArmWithAnExtraSlotHoldsTheRecordReadOnly() - { - var arm = VectorFiles.Committed("union-parity", "extra-slot-unit-arm.msgpack"); - var payload = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("u"); w.WriteRaw(arm); }); - var path = Seed(UnitRegistry, typeof(UnitDoc), () => new UnitDoc { Name = "u" }, payload); - using var cache = Open(path, UnitRegistry); - var unread = Stored(cache, Marked).UnreadSlots; - Assert.That(unread, Is.Not.Null); - Assert.That(unread!.TypeName, Is.EqualTo(nameof(UnionParityVectorTests.Denied2))); - Assert.That(unread.SlotCount, Is.EqualTo(1)); - Assert.That(unread.DeclaredCount, Is.EqualTo(0)); - Assert.That(cache.IsWritable(Marked), Is.False); - Assert.That(cache.IsWritable(Clean), Is.True); - } - // ---- every staged write and removal of a marked key is refused, nothing else is ---- private static void AssertSlotRefusal(TestDelegate act, string what, string type, int slots, int declared) @@ -194,69 +153,73 @@ namespace GameCult.Caching.Tests AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "removal", "UnionSurface", 8, 7); } + // Every row of slot-outcomes.msgpack, from the committed files alone: the mark, Explained and IsWritable of each record of + // extra-slot-store.cc and extra-slot-consent.cc, and a removal of the key refused exactly where IsWritable is false. [Test] - public void AListedCompatibleIdDoesNotConsentToANestedMark() + public void EverySlotOutcomeRowHoldsAndARemovalIsRefusedExactlyWhereTheRecordIsNotWritable() { - // The record's stored id is one the reader lists compatible; the extra slot is nested, so the declaration does not reach it. - var path = SeedSurface("extra-slot-nested-record.msgpack"); - Relabel(path, OldId); - using var cache = Open(path, DeclaringRegistry); - Assert.That(Stored(cache, Marked).StoredSchemaId, Is.EqualTo(OldId)); - Assert.That(cache.IsWritable(Marked), Is.False); - AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Marked))), "nested under a listed id", nameof(UnionParityVectorTests.Composition), 3, 2); - Assert.That(cache.IsWritable(Clean), Is.True, "the clean record under the listed id is consented to (declared)"); - Assert.That(cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Clean))), Is.True); - } - - // The record stored under a listed compatible id whose catalog entry has one more member than the reader (slot 7), the payload - // carrying that slot at depth 0: the loss a declaration consents to. With a nested mark beside it, nothing is consented to. - [TestCase("union-surface.msgpack", true, TestName = "ADepthZeroSlotTheCatalogReportsUnderAListedIdIsConsentedTo")] - [TestCase("extra-slot-nested-record.msgpack", false, TestName = "ANestedMarkBesideACatalogReportedSlotIsNotConsentedTo")] - public void ACompatibleIdConsentsOnlyToTheDepthZeroSlotTheCatalogReports(string vector, bool consented) - { - var original = VectorFiles.Committed("union-parity", vector); - var path = Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, AppendRootSlot(original, "wider writer")); - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); - var entry = snapshot.SchemaCatalog.Single(); - var last = entry.Members.OrderBy(member => member.Slot).Last(); - entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 7, MemberName = "Wider", TypeName = last.TypeName }).ToArray(); - entry.SchemaId = OldId; - foreach (var record in snapshot.Records) record.SchemaId = OldId; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); - - using var cache = Open(path, DeclaringRegistry); - var unread = Stored(cache, Marked).UnreadSlots; - Assert.That(unread, Is.Not.Null); - Assert.That(unread!.Explained, Is.EqualTo(consented)); - Assert.That(cache.IsWritable(Marked), Is.EqualTo(consented)); - if (consented) - Assert.That(cache.Commit(batch => batch.Upsert(new SurfaceDeclaring { Id = "new" }, new CultRecordHandle(Marked))), Is.True); - else - Assert.Throws(() => cache.Commit(batch => batch.Remove(Marked))); - } - - // The payload's root array with one string slot appended. - private static byte[] AppendRootSlot(byte[] payload, string value) - { - var reader = new MessagePackReader(payload); - var count = reader.ReadArrayHeader(); - var start = (int)reader.Consumed; - for (var i = 0; i < count; i++) reader.Skip(); - var end = (int)reader.Consumed; - return Emit((ref MessagePackWriter w) => + var reader = new MessagePackReader(VectorFiles.Committed("union-parity", "slot-outcomes.msgpack")); + var rows = reader.ReadArrayHeader(); + Assert.That(rows, Is.EqualTo(7)); + for (var i = 0; i < rows; i++) { - w.WriteArrayHeader(count + 1); - w.WriteRaw(payload.AsSpan(start, end - start)); - w.Write(value); - }); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(8)); + var store = reader.ReadString()!; + var key = new CultRecordKey(reader.ReadString()!); + var writable = reader.ReadBoolean(); + bool? explained = reader.TryReadNil() ? null : reader.ReadBoolean(); + var type = reader.TryReadNil() ? null : reader.ReadString(); + var counts = new int?[3]; + for (var c = 0; c < 3; c++) counts[c] = reader.TryReadNil() ? null : reader.ReadInt32(); + var (slots, declared, occurrences) = (counts[0], counts[1], counts[2]); + var row = $"{store} {key.Value}"; + + var path = Path.Combine(_directory, $"{i}-{store}"); + File.WriteAllBytes(path, VectorFiles.Committed("union-parity", store)); + var before = File.ReadAllBytes(path); + using var cache = Open(path, UnionParityVectorTests.SlotReader(store)); + var unread = Stored(cache, key).UnreadSlots; + if (explained == null) + { + Assert.That(unread, Is.Null, row); + } + else + { + Assert.That(unread, Is.Not.Null, row); + Assert.That((unread!.Explained, unread.TypeName, unread.SlotCount, unread.DeclaredCount, unread.Occurrences), + Is.EqualTo((explained.Value, type, slots, declared, occurrences)), row); + } + + Assert.That(cache.IsWritable(key), Is.EqualTo(writable), row); + Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable"); + if (writable) + { + Assert.That(cache.Commit(batch => batch.Remove(key)), Is.True, row); + } + else + { + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Remove(key)), row)!; + Assert.That(refusal.UnreadSlots, Is.Not.Null, row); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused removal leaves the store as it was"); + } + } } - private static void Relabel(string path, string id) + // Consent reaches only the slots the catalog comparison reports: the wide id's catalog reports slot 7, so a payload carrying + // slots 7 and 8 at depth 0 is not consented to, and a payload carrying slot 7 alone is. + [Test] + public void ConsentStopsAtTheSlotsTheCatalogReports() { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); - snapshot.SchemaCatalog.Single().SchemaId = id; - foreach (var record in snapshot.Records) record.SchemaId = id; - File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var path = Path.Combine(_directory, "extra-slot-consent.cc"); + File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-consent.cc")); + var before = File.ReadAllBytes(path); + using var cache = Open(path, UnionParityVectorTests.SlotReader("extra-slot-consent.cc")); + var over = new CultRecordHandle(new CultRecordKey("over-consent")); + AssertWriteAgrees(cache, over.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "over" }, over))); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the refused write leaves the store as it was"); + var consented = new CultRecordHandle(new CultRecordKey("consented")); + AssertWriteAgrees(cache, consented.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "consented" }, consented))); + Assert.That(cache.IsWritable(consented.Key), Is.True); } // ---- carried verbatim ---- @@ -349,5 +312,241 @@ namespace GameCult.Caching.Tests Assert.That(reader.IsWritable(Marked), Is.False, "the wide id is undeclared: unowned, though the mark is explained"); Assert.Throws(() => reader.Commit(batch => batch.Remove(Marked))); } + + // ---- declared slots are the slots the type's own formatter writes ---- + + // MessagePack writes a non-public [Key] member of an AllowPrivate type: 2 slots, where a public-member count says 1. + [MessagePackObject(AllowPrivate = true)] + public sealed class PrivInner + { + [Key(0)] public string A = string.Empty; + [Key(1)] internal string B = string.Empty; + } + + // A [Key] getter that throws on an uninitialized instance; the parameterless constructor's instance is written instead. + [MessagePackObject] + public sealed class Counted + { + private List _items = new(); + [Key(0)] public int Count { get => _items.Count; set => _items = Enumerable.Repeat(string.Empty, value).ToList(); } + } + + // The same getter and no parameterless constructor: no instance can be written, so the guard leaves the type alone. + [MessagePackObject] + public sealed class Uncounted + { + private readonly List _items; + [SerializationConstructor] public Uncounted(int count) => _items = Enumerable.Repeat(string.Empty, count).ToList(); + [Key(0)] public int Count => _items.Count; + } + + // A type naming its own formatter owns its layout: one [Key] slot, but the formatter writes the item count and each item. + [MessagePackObject] + [MessagePackFormatter(typeof(BagFormatter))] + public sealed class Bag + { + [Key(0)] public List Items = new(); + } + + public sealed class BagFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, Bag? value, MessagePackSerializerOptions options) + { + if (value == null) { writer.WriteNil(); return; } + writer.WriteArrayHeader(1 + value.Items.Count); + writer.Write(value.Items.Count); + foreach (var item in value.Items) writer.Write(item); + } + + public Bag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + if (reader.TryReadNil()) return null; + var count = reader.ReadArrayHeader(); + reader.ReadInt32(); + var bag = new Bag(); + for (var i = 1; i < count; i++) bag.Items.Add(reader.ReadString()!); + return bag; + } + } + + [CultDocument("tests.slot_measured", "tests.slot_measured.v1")] + [MessagePackObject] + public sealed class MeasuredDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public PrivInner? Private; + [Key(2)] public Counted? Counted; + [Key(3)] public Uncounted? Uncounted; + [Key(4)] public Bag? Bag; + } + + private static readonly CultDocumentRegistry MeasuredRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(MeasuredDoc) }); + + private static MeasuredDoc Measured(string member) => member switch + { + "private" => new MeasuredDoc { Name = member, Private = new PrivInner { A = "a", B = "bee" } }, + "constructed" => new MeasuredDoc { Name = member, Counted = new Counted { Count = 2 } }, + "unmeasurable" => new MeasuredDoc { Name = member, Uncounted = new Uncounted(2) }, + "own-formatter" => new MeasuredDoc { Name = member, Bag = new Bag { Items = { "x", "y", "z" } } }, + _ => throw new ArgumentOutOfRangeException(nameof(member), member, null) + }; + + [TestCase("private", TestName = "APrivateKeyedMemberDoesNotMark")] + [TestCase("constructed", TestName = "ATypeMeasuredThroughItsConstructorDoesNotMark")] + [TestCase("unmeasurable", TestName = "AnUnmeasurableTypeIsNotGuarded")] + [TestCase("own-formatter", TestName = "ATypeWithItsOwnFormatterIsNotMeasuredByKeys")] + public void ARecordWrittenByItsOwnTypeDoesNotMark(string member) + { + var path = Path.Combine(_directory, "measured.cc"); + using (var cache = Open(path, MeasuredRegistry)) + { + Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle(Marked))), Is.True); + } + + using (var cache = Open(path, MeasuredRegistry)) + { + Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, member); + Assert.That(cache.IsWritable(Marked), Is.True, member); + Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle(Marked))), Is.True, member); + Assert.That(cache.Commit(batch => batch.Remove(Marked)), Is.True, member); + } + } + + // One slot past what the formatter writes still marks, with the written count as declared. + [Test] + public void AnExtraSlotPastTheWrittenCountStillMarks() + { + var privatePayload = Emit((ref MessagePackWriter w) => + { + w.WriteArrayHeader(5); w.Write("p"); + w.WriteArrayHeader(3); w.Write("a"); w.Write("bee"); w.Write(Extra); + w.WriteNil(); w.WriteNil(); w.WriteNil(); + }); + var path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, privatePayload, "private.cc"); + using (var cache = Open(path, MeasuredRegistry)) + { + AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "AllowPrivate type", nameof(PrivInner), 3, 2); + Assert.That(cache.IsWritable(Clean), Is.True); + } + + var countedPayload = Emit((ref MessagePackWriter w) => + { + w.WriteArrayHeader(5); w.Write("c"); w.WriteNil(); + w.WriteArrayHeader(2); w.Write(0); w.Write(Extra); + w.WriteNil(); w.WriteNil(); + }); + path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, countedPayload, "counted.cc"); + using (var cache = Open(path, MeasuredRegistry)) + AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "measured through its constructor", nameof(Counted), 2, 1); + } + + // ---- a flatten writes its base chain's decoded view: refused when a record on the chain is not owned ---- + + private static readonly CultRecordKey Base = new("base"); + private static readonly CultRecordKey Var = new("var"); + private static readonly CultRecordKey Fresh = new("fresh"); + + // A store holding plain record base and its variant var, base then given the payload. + private string SeedVariant(byte[]? basePayload, string name) + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, SurfaceRegistry)) + { + Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "base" }, new CultRecordHandle(Base))), Is.True); + Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True); + } + + if (basePayload == null) return path; + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.Records.Single(record => record.Key == Base.Value).Payload = basePayload; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // A variant overrides its name. + private static CultVariantOverride[] Named(CultCache cache, CultRecordKey key) => new[] { cache.Override(nameof(Surface.Id), key.Value) }; + + private static void AssertBaseRefusal(TestDelegate act, string what, bool unreadSlots) + { + var refusal = Assert.Throws(act, what)!; + Assert.That(refusal.RecordKey, Is.EqualTo(Base.Value), $"{what}: the refusal is the base's own"); + Assert.That(refusal.UnreadSlots, unreadSlots ? Is.Not.Null : Is.Null, what); + } + + [Test] + public void AFlattenThroughABaseHeldForUnreadSlotsIsRefusedAndTheVariantStillWritesItsDelta() + { + var nested = VectorFiles.Committed("union-parity", "extra-slot-nested-record.msgpack"); + var path = SeedVariant(nested, "flatten.cc"); + var before = File.ReadAllBytes(path); + using (var cache = Open(path, SurfaceRegistry)) + { + Assert.That(cache.IsWritable(Base), Is.False); + Assert.That(cache.IsWritable(Var), Is.True, "the variant persists only its delta"); + AssertBaseRefusal(() => cache.Commit(batch => batch.Flatten(Var)), "Commit(Flatten)", unreadSlots: true); + AssertBaseRefusal(() => cache.FlattenAsync(Var).GetAwaiter().GetResult(), "FlattenAsync", unreadSlots: true); + AssertBaseRefusal(() => cache.Commit(batch => { batch.UpsertVariant(Fresh, Base, Named(cache, Fresh)); batch.Flatten(Fresh); }), "a new variant flattened as it lands", unreadSlots: true); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused flattens leave the store as it was"); + Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True, "a variant write persists only its delta"); + Assert.That(cache.Commit(batch => batch.Remove(Var)), Is.True); + } + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Assert.That(after.Records.Single(record => record.Key == Base.Value).Payload, Is.EqualTo(nested), "the base's bytes stay"); + } + + [Test] + public void AFlattenThroughABaseUnderAnUndeclaredDroppingIdIsRefused() + { + var path = Path.Combine(_directory, "drift.cc"); + using (var writer = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) }))) + Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "base", Extra = "kept?" }, new CultRecordHandle(Base))), Is.True); + + using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) })); + Assert.That(reader.IsWritable(Base), Is.False); + Assert.That(reader.Commit(batch => batch.UpsertVariant(Var, Base, new[] { reader.Override(nameof(NarrowDoc.Name), Var.Value) })), Is.True, "a variant over the unowned base persists only its delta"); + var before = File.ReadAllBytes(path); + AssertBaseRefusal(() => reader.Commit(batch => batch.Flatten(Var)), "flatten over undeclared drift", unreadSlots: false); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(reader.Commit(batch => batch.Remove(Var)), Is.True); + } + + // ---- IsWritable is the write's own answer ---- + + // False exactly where the write throws. + private static void AssertWriteAgrees(CultCache cache, CultRecordKey key, Action write) + { + var writable = cache.IsWritable(key); + Exception? thrown = null; + try { write(); } + catch (InvalidOperationException exception) { thrown = exception; } + Assert.That(thrown == null, Is.EqualTo(writable), $"IsWritable({key.Value}) is {writable}; the write {(thrown == null ? "landed" : "threw: " + thrown.Message)}"); + } + + [Test] + public void IsWritableIsFalseInAReadOnlyStoreAndAFlattenThereIsNotRefusedForItsCleanBase() + { + var path = SeedVariant(null, "readonly.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = SurfaceRegistry, ReadOnly = true }); + Assert.That(cache.IsWritable(Base), Is.False); + AssertWriteAgrees(cache, Base, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle(Base)))); + AssertWriteAgrees(cache, Var, () => cache.Commit(batch => batch.Remove(Var))); + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Flatten(Var)))!; + Assert.That(refusal, Is.Not.InstanceOf(), "nothing on the chain is lost: the refusal is the store's read-only one"); + Assert.That(refusal.Message, Does.Contain("read-only backing store")); + } + + [Test] + public void IsWritableIsFalseForARecordNoAttachedStoreIsHomeTo() + { + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Surface), typeof(GapDoc) })); + Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "memory" }, new CultRecordHandle(Clean))), Is.True); + Assert.That(cache.IsWritable(Clean), Is.True, "a store-less cache writes its own records"); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(Path.Combine(_directory, "gaps.cc")), typeof(GapDoc)); + Assert.That(cache.IsWritable(Clean), Is.False); + AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "x" }, new CultRecordHandle(Clean)))); + AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Remove(Clean))); + Assert.That(cache.IsWritable(Unheld), Is.True, "an unheld key no store lists foreign is writable"); + } } } diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 22ef82c9..30e8f744 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -252,7 +252,7 @@ namespace GameCult.Caching.Tests private static Emit ImageArm(string id, string uri, string alt) => ArmOf(5, Str(id), Str(uri), Nil, Str(alt)); // The constructed value's payload, with a part of the structure replaceable to build a refusal or an extra-slot case. - private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, Emit? trailing = null) + private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, params Emit[] trailing) { extra ??= new Extra(); root ??= BoxArm("0000000000c0", Slots( @@ -264,7 +264,7 @@ namespace GameCult.Caching.Tests outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3"))); crafted ??= ArmOf(1, Str("gear"), Num(4)); var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil }; - return Raw(Slots(trailing != null ? slots.Append(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots)); + return Raw(Slots(trailing.Length > 0 ? slots.Concat(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots)); } private static byte[] SerializeValue(UnionSurface value) => CultDocumentMessagePackSerialization.SerializeUntyped(value, typeof(UnionSurface), Registry); @@ -342,6 +342,70 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class Floor2 : UnitOutcome { } [MessagePackObject] public sealed class Ceiling2 : UnitOutcome { } + // The document a unit arm is placed in for a store: [Name, Outcome]. + [CultDocument("tests.slot_unit", "tests.slot_unit.v1")] + [MessagePackObject] + public sealed class UnitDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public UnitOutcome? Outcome; + } + + // ---- the slot stores: extra-slot-store.cc, extra-slot-consent.cc and their outcomes as data ---- + + // The id extra-slot-consent.cc's records are stored under: UnionSurface's catalog entry with one more member, slot 7. + internal const string WideId = "tests.union_surface.wide"; + + // UnionSurface's members under a new version, listing the wide id compatible: the consent vector's reader. + [CultDocument("tests.union_surface", "tests.union_surface.v2", CompatibleSchemaIds = new[] { WideId })] + [MessagePackObject] + public sealed class SurfaceDeclaring + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public int Version; + [Key(2)] public Node? Root; + [Key(3)] public List Outcomes = new(); + [Key(4)] public Item? Part; + [Key(5)] public CraftedItem? Crafted; + [Key(6)] public Node? Missing; + } + + // The registry each slot store is read with. + internal static CultDocumentRegistry SlotReader(string store) => store switch + { + "extra-slot-store.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(UnionSurface), typeof(UnitDoc) }), + "extra-slot-consent.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(SurfaceDeclaring) }), + _ => throw new ArgumentOutOfRangeException(nameof(store), store, null) + }; + + // One row of slot-outcomes.msgpack: what the store's reader holds at the key. The mark fields are null for an unmarked record. + private sealed record SlotOutcome(string Store, string Key, bool Writable, bool? Explained, string? Type, int? Slots, int? Declared, int? Occurrences); + + private static readonly SlotOutcome[] SlotOutcomes = + { + new("extra-slot-store.cc", "marked", false, false, nameof(Composition), 3, 2, 1), + new("extra-slot-store.cc", "clean", true, null, null, null, null, null), + new("extra-slot-store.cc", "unit-arm", false, false, nameof(Denied2), 1, 0, 1), + new("extra-slot-consent.cc", "consented", true, true, nameof(SurfaceDeclaring), 8, 7, 1), + new("extra-slot-consent.cc", "over-consent", false, false, nameof(SurfaceDeclaring), 9, 7, 1), + new("extra-slot-consent.cc", "nested", false, false, nameof(SurfaceDeclaring), 8, 7, 2), + new("extra-slot-consent.cc", "clean", true, null, null, null, null, null), + }; + + private static byte[] SlotOutcomesFile() => Raw((ref MessagePackWriter w) => + { + w.WriteArrayHeader(SlotOutcomes.Length); + foreach (var row in SlotOutcomes) + { + w.WriteArrayHeader(8); + w.Write(row.Store); w.Write(row.Key); w.Write(row.Writable); + if (row.Explained is { } explained) w.Write(explained); else w.WriteNil(); + if (row.Type is { } type) w.Write(type); else w.WriteNil(); + foreach (var count in new[] { row.Slots, row.Declared, row.Occurrences }) + if (count is { } value) w.Write(value); else w.WriteNil(); + } + }); + private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly); // The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes. @@ -417,27 +481,28 @@ namespace GameCult.Caching.Tests } } - // The store the slot-readonly tests open and the other runtimes read: the extra-slot-nested-record payload as the record "marked" - // beside the ordinary value as "clean". The store's clock is pinned as union-surface.cc's. - [Test] - public void ExtraSlotStoreVectorHoldsAMarkedAndACleanRecord() + // A single-file store of the records, each written as its type's value with the store's clock pinned as union-surface.cc's, then + // its payload replaced by bytes no current writer produces; edit changes the snapshot further before it is serialized. + private static byte[] SlotStore(CultDocumentRegistry registry, (string Key, object Value, byte[] Payload)[] records, Action? edit = null) { var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}"); Directory.CreateDirectory(work); try { - var scratch = Path.Combine(work, "extra-slot-store.cc"); - using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry, StoreFlushOnDispose = true })) + var scratch = Path.Combine(work, "slots.cc"); + using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = registry, StoreFlushOnDispose = true })) { cache.Clock = () => StoredAt; - cache.UpsertAsync(typeof(UnionSurface), Constructed(), new CultRecordKey("marked")).GetAwaiter().GetResult(); - cache.UpsertAsync(typeof(UnionSurface), Constructed(), new CultRecordKey("clean")).GetAwaiter().GetResult(); + foreach (var (key, value, _) in records) + cache.UpsertAsync(value.GetType(), value, new CultRecordKey(key)).GetAwaiter().GetResult(); cache.FlushAsync().GetAwaiter().GetResult(); } var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(scratch)); - snapshot.Records.Single(record => record.Key == "marked").Payload = Committed("extra-slot-nested-record.msgpack"); - Check("extra-slot-store.cc", CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + foreach (var (key, _, payload) in records) + snapshot.Records.Single(record => record.Key == key).Payload = payload; + edit?.Invoke(snapshot); + return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); } finally { @@ -445,6 +510,44 @@ namespace GameCult.Caching.Tests } } + // The store the slot-readonly tests open and the other runtimes read: "marked" holds the extra-slot-nested-record payload, + // "clean" the ordinary value, and "unit-arm" a tests.slot_unit document ["unit-arm", [2, ["extra-slot"]]]. + [Test] + public void ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord() + { + Check("extra-slot-store.cc", SlotStore(SlotReader("extra-slot-store.cc"), new (string, object, byte[])[] + { + ("marked", Constructed(), Committed("extra-slot-nested-record.msgpack")), + ("clean", Constructed(), Surface()), + ("unit-arm", new UnitDoc { Name = "unit-arm", Outcome = new Denied2() }, Raw(Slots(Str("unit-arm"), ArmOf(2, ExtraSlot)))), + })); + } + + // The consent store: every record under the wide id, whose catalog entry is UnionSurface's 7 members plus slot 7. Its reader + // lists the wide id compatible, so the comparison reports slot 7 ignored; only a depth-0 mark within that slot is consented to. + [Test] + public void ExtraSlotConsentVectorHoldsTheConsentCases() + { + Check("extra-slot-consent.cc", SlotStore(Registry, new (string, object, byte[])[] + { + ("consented", Constructed(), Surface(trailing: ExtraSlot)), + ("over-consent", Constructed(), Surface(trailing: new[] { ExtraSlot, ExtraSlot })), + ("nested", Constructed(), Surface(extra: new Extra(Composition: true), trailing: ExtraSlot)), + ("clean", Constructed(), Surface()), + }, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + var last = entry.Members.OrderBy(member => member.Slot).Last(); + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 7, MemberName = "Wider", TypeName = last.TypeName }).ToArray(); + entry.SchemaId = WideId; + foreach (var record in snapshot.Records) record.SchemaId = WideId; + })); + } + + // The outcomes of both slot stores as data; SlotReadOnlyTests asserts every row from the committed files. + [Test] + public void SlotOutcomesVectorMatches() => Check("slot-outcomes.msgpack", SlotOutcomesFile()); + // The value names every arm shape the vectors exist to hold other runtimes to. [Test] public void ConstructedValueExercisesEveryArmShape() @@ -816,7 +919,8 @@ namespace GameCult.Caching.Tests Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x91 }), $"switch {off ?? "unset"} rewrites nothing"); File.Delete(path); - foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles }) + foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles, + ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord, ExtraSlotConsentVectorHoldsTheConsentCases, SlotOutcomesVectorMatches }) body(); } diff --git a/tests/vectors/union-parity/extra-slot-consent.cc b/tests/vectors/union-parity/extra-slot-consent.cc new file mode 100644 index 0000000000000000000000000000000000000000..54ed4d954290e45dce66044d4099e1fc40c73d09 GIT binary patch literal 3479 zcmeH}&2G~`5XbvTFTHYyT+_PtZtTPnLP9xIk)WWqlmkL&y&gA~9J{hNZ6nloph={L z8y{%{s9Zoo9HSy0B0LR~*ma#!kkZhILlaAhcV}jQGydz{eW5uEW7NP+Ntkhbam z!8ZYc2&9T@d+zg~;#-V^v}2p_<${T?7mP{UN_ROZF`{aDL!lhMYR067!Zc#(L^ZPp zrI0f$s?sdbtfc`l%$&v*4UGsWaxSM6rpuJFpR@BXQUL;%*$OK|DxbPw!PG)zvH^a8 z)E8SJ78;hJY@k&t{}kgoo=D~8g_IkHE}Cc{s$0Ef)~bPkva}|A+ug)rJ6Ht^d~i9= zTFeOmV==cH-w5V!f>=wSx)sMkN7sbidydU;&1fVDQxSdz_7+Ytlo;bQi}Nf+S5R^_ zDE)-aSV+vW>S2jukY%h>0VN|9P%~H;X}vF%$33Y$V!kb^8^BRW{o(YAXTn%;X^i`@ zBDhzE;0!r>js?$g5Dhs4X&GGbZ7$un194}#l%n`|$C6eiP77;$>8DUc8IJ{5jd2S; zvjikI({M$O+{Xs4#o!O@APBd6)CGnEziPoh+uM&DE9gv9HBd9ekaK`NJfj%8rDx0> zR~2e-;HGKFOa`JdqGYM5>xQgkEt4?LRjM_c6K{*=r-3$ia?<(UX->AI26P-v_WKUb zophR0tzJXulHuzT`cJ<<-J#u$tJ#_1Iy0@_mKm0_r?}96wjGJEs}GlILc3w48|_F` z!@~5$6o!~Pe1fO1f9U{?#vylrxqm#^ySIruB5BVYWyt!W6`qOAe6Q)PC}wqQOy#mCNAI z6X|ic6riCW{!$U86R=Yy` z0D>Zjs4q~e?;!OczJ@nbs730rw;ASRe&2laJ$trrIa6;myK7+yiOsS$vpoQg3{7-E z6kMb(aePExpa_E6K;nr&A6-inh-G`e4HT#Z&Isr1F_K{v8sUC5${$q9(eqGm9KwJ_ zI5B;&yt>6L6dnhQ+LBTRp6UA%K!6Y~HHi=oL&g{dGqz-U4shaw@Wh9vPW8|&$pAMn zka|`hqUkIB@s6y8BKD&zU0Y1`7N}M2JiU1kxz)J7NI8?+@ddyAUUFA`jZYOv zO{mctyXnCmawlOV10xsJ%9UcL(|&EYwA((dD-^XTIkLqT3?zKz+<;e_S diff --git a/tests/vectors/union-parity/slot-outcomes.msgpack b/tests/vectors/union-parity/slot-outcomes.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..eed3290487d9068da5b702999bf0ce790bd099de GIT binary patch literal 322 zcmZ{f!3}~y6hs9NU@_~#7Ni3!*eqYO5&k!SKgLd^0a2p|6XV_NHl`#TNPs!bVP0nD zT)*KYtd%QBloMiD8q<`T_mF%4SW|f*XT=iE<7hN|F$HK0-&Ui6TZLAX=F^ok9k1C)RDy|a*q@Y+-~USOp* Date: Thu, 12 Nov 2026 00:30:37 +0000 Subject: [PATCH 08/20] CultUnionGuard: declared slots are the array the type's own formatter writes; [MessagePackFormatter] types are not guarded DeclaredSlots (the reflected public [Key] count) is deleted: it falsely marked an AllowPrivate type's own write. SlotFormatter measures one instance (uninitialized, else a parameterless constructor) on its first read inside a scope and keeps the count; a type neither instance can be written for is not guarded (question slot-unmeasurable-type, recommended answer unguarded). Co-Authored-By: Claude Opus 5.5 --- .../CultUnionGuard.cs | 65 +++++++++++++------ src/GameCult.Caching/CultUnreadSlots.cs | 2 +- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs index 6fb2e004..1a2a535e 100644 --- a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs +++ b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs @@ -1,7 +1,9 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using MessagePack; using MessagePack.Formatters; @@ -19,33 +21,30 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol { var arms = typeof(T).GetCustomAttributes(inherit: false).ToArray(); if (arms.Length == 0) - return DeclaredSlots(typeof(T)) is { } declared && inner.GetFormatter() is { } objectFormatter - ? new SlotFormatter(objectFormatter, declared) - : null; + return Guarded(typeof(T)) && inner.GetFormatter() is { } objectFormatter ? new SlotFormatter(objectFormatter) : null; return inner.GetFormatter() is not { } formatter ? null : new Formatter(formatter, arms); } - // The slots a concrete int-keyed [MessagePackObject] type declares (highest [Key] plus one), by MessagePack's member rule: public - // instance properties and fields, base keys included, [IgnoreMember] excluded. Null for any other type, which the guard leaves alone. - private static int? DeclaredSlots(Type type) - { - if (type.IsAbstract || type.IsInterface || type.IsGenericTypeDefinition) return null; - if (type.GetCustomAttribute(inherit: true) is not { KeyAsPropertyName: false }) return null; - var highest = -1; - const BindingFlags members = BindingFlags.Public | BindingFlags.Instance; - foreach (var member in type.GetFields(members).Cast().Concat(type.GetProperties(members))) - { - if (member.GetCustomAttribute(inherit: true) != null) continue; - if (member.GetCustomAttribute(inherit: true)?.IntKey is { } key) highest = Math.Max(highest, key); - } - - return highest + 1; - } + // A concrete int-keyed [MessagePackObject] type whose layout MessagePack derives from its [Key] members. A type that names its own + // [MessagePackFormatter] owns its layout and its tolerance (it may read a legacy shape or skip fields on purpose), so it is not guarded. + private static bool Guarded(Type type) => + !type.IsAbstract && !type.IsInterface && !type.IsGenericTypeDefinition && + type.GetCustomAttribute(inherit: true) is { KeyAsPropertyName: false } && + type.GetCustomAttribute(inherit: true) == null; // Counts a concrete object's array header against its declared slots and notes the excess into the open CultSlotScope, then lets the // inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through. - private sealed class SlotFormatter(IMessagePackFormatter inner, int declared) : IMessagePackFormatter + // Declared slots are what the type's own formatter writes: the array header of one instance written through it, measured on the + // first read inside a scope (not in GetFormatter, so a self-referential type never recurses into its own resolver cache) and kept. + private sealed class SlotFormatter(IMessagePackFormatter inner) : IMessagePackFormatter { + private const int NotMeasured = -1; + + // A type no instance of which can be written is not guarded: no count can exceed this. + private const int Unguarded = int.MaxValue; + + private int declared = NotMeasured; + public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options) => inner.Serialize(ref writer, value, options); @@ -57,6 +56,7 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol if (!peek.TryReadNil() && peek.NextMessagePackType == MessagePackType.Array) { var count = peek.ReadArrayHeader(); + if (declared == NotMeasured) declared = Measure(options); if (count > declared) scope.Note(typeof(T).Name, count, declared); } @@ -70,6 +70,31 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol scope.Depth--; } } + + // The uninitialized instance, else one from a parameterless constructor (public or not) when that write throws. + private int Measure(MessagePackSerializerOptions options) + { + if (Written(() => (T)RuntimeHelpers.GetUninitializedObject(typeof(T)), options) is { } slots) return slots; + var constructor = typeof(T).GetConstructor(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, null, Type.EmptyTypes, null); + return constructor != null && Written(() => (T)constructor.Invoke(null), options) is { } constructed ? constructed : Unguarded; + } + + private int? Written(Func instance, MessagePackSerializerOptions options) + { + try + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + inner.Serialize(ref writer, instance(), options); + writer.Flush(); + var written = new MessagePackReader(buffer.WrittenMemory); + return written.NextMessagePackType == MessagePackType.Array ? written.ReadArrayHeader() : null; + } + catch (Exception) + { + return null; + } + } } private sealed class Formatter(IMessagePackFormatter inner, UnionAttribute[] arms) : IMessagePackFormatter diff --git a/src/GameCult.Caching/CultUnreadSlots.cs b/src/GameCult.Caching/CultUnreadSlots.cs index f6d0416a..cbf46f3c 100644 --- a/src/GameCult.Caching/CultUnreadSlots.cs +++ b/src/GameCult.Caching/CultUnreadSlots.cs @@ -24,7 +24,7 @@ namespace GameCult.Caching /// The slots that object carried. public int SlotCount { get; } - /// The slots this build declares for that type (highest key plus one). + /// The slots this build declares for that type: the array its own formatter writes for one instance. public int DeclaredCount { get; } /// The objects in the record that carried extra slots. -- 2.47.3 From 74f1db64afa642bcb770500f9034e5a4d26347ec Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 00:31:01 +0000 Subject: [PATCH 09/20] CultCache: a mark is explained only up to the highest slot the catalog comparison reports Explained required only that some slot was reported, so a depth-0 slot the comparison never reported was consented to and dropped on write (finding unreported-slot-over-consent). Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 7 +++++-- src/GameCult.Caching/CultUnreadSlots.cs | 7 ++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index da791b82..ebea77d4 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -4034,8 +4034,11 @@ namespace GameCult.Caching for (var cause = exception; cause != null; cause = cause.InnerException) causes.Add(cause.Message); throw new InvalidDataException($"Record '{record.Key}' (schema '{record.SchemaId}') cannot be decoded: {string.Join(" <- ", causes.Distinct())}", exception); } - // Explained only when every mark is at depth 0 and the schema comparison reported ignored slots under this id. - unread = decoded.UnreadSlots?.Explaining(decoded.UnreadSlots.AllAtDepthZero && report.IgnoredExtraSlots.Length > 0); + // Explained only when every mark is at depth 0 and the schema comparison under this id reported every slot the mark carries + // past the reader: consent reaches no slot above the highest reported (none when nothing was reported). + unread = decoded.UnreadSlots is { } marked + ? marked.Explaining(marked.AllAtDepthZero && marked.SlotCount <= report.IgnoredExtraSlots.DefaultIfEmpty(-1).Max() + 1) + : null; return new CultStoredDocument( new CultRecordKey(record.Key), record.StoredAt, diff --git a/src/GameCult.Caching/CultUnreadSlots.cs b/src/GameCult.Caching/CultUnreadSlots.cs index cbf46f3c..9e8412f7 100644 --- a/src/GameCult.Caching/CultUnreadSlots.cs +++ b/src/GameCult.Caching/CultUnreadSlots.cs @@ -31,9 +31,10 @@ namespace GameCult.Caching public int Occurrences { get; } /// - /// True only when every mark is at depth 0 and the schema comparison reported ignored slots under the id the record was stored - /// under: the loss a compatible-id declaration consents to (cultlib-gaps:ruling:slot-mark-consent). Any other mark leaves the - /// record read-only whatever is declared. + /// True only when every mark is at depth 0 and the schema comparison under the id the record was stored under reported every + /// slot the mark carries past the reader ( is at most the highest reported slot plus one): the loss a + /// compatible-id declaration consents to (cultlib-gaps:ruling:slot-mark-consent). Any other mark leaves the record read-only + /// whatever is declared. /// public bool Explained { get; } -- 2.47.3 From 4fca621ec0d510ea50d4deef4278f4462eac6ca9 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 00:31:48 +0000 Subject: [PATCH 10/20] CultCache: one owner decides whether a record is writable; a flatten through an unowned base is refused WriteRefusal holds Claim's whole decision (no home store, read-only home, the home store's Refusal) and the flatten-chain walk: a flatten is refused with the refusal of the first record on its base chain its home store does not own. Claim throws its answer; IsWritable asks it of the held record's removal form, and for an unheld key whether no store lists it foreign. IsWritable's per-store Refusal sweep is deleted (findings iswritable-readonly-store, flatten-marked-base-loss). MintElementIds' doc names the mark. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 80 +++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index ebea77d4..b386be0c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2308,8 +2308,9 @@ namespace GameCult.Caching // The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in // memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records - // it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only, - // so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand. + // it rewrote; a second call finds none. A selected record that is not owned (neither declared nor read whole, or held + // read-only for slots past this build's, see CultStoredDocument.UnreadSlots) is read-only, so its home store's commit is + // refused with CultSchemaConflictException until a build that reads those slots rewrites it; commits to earlier stores stand. public int MintElementIds() { var groups = Held(() => _entries.Values @@ -2665,14 +2666,19 @@ namespace GameCult.Caching } /// - /// Whether a write or removal of the key would be let through by every attached store: false when a store lists the key foreign, - /// or holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's, see - /// ); true for a key no store holds. Live and advisory: the write still decides. + /// Whether a removal of the key, or a write of the record's own type, would be let through: the write's own decision, asked of + /// the record the cache holds. False when no attached store is home to the record's type, when its home store is read-only, or + /// when that store holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's, + /// see ). For a key the cache does not hold, false only when a store lists it + /// foreign. One write it does not predict: a Flatten of a variant whose base chain holds an unowned record is refused, though + /// the variant itself is writable. Live and advisory: the write still decides. /// public bool IsWritable(CultRecordKey key) { lock (Gate()) - return _stores.All(entry => entry.Store.Refusal(key.Value, null) == null); + return _entries.TryGetValue(key.Value, out var held) + ? WriteRefusal(held, upsert: false, null, out _) == null + : _stores.All(entry => entry.Store.ForeignRecords.All(record => !string.Equals(record.Key, key.Value, StringComparison.Ordinal))); } public object? Get(CultRecordKey key) @@ -3170,6 +3176,8 @@ namespace GameCult.Caching IReadOnlyCollection? removed) { var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal); + var batch = new Dictionary(StringComparer.Ordinal); + foreach (var stored in admitted) batch[stored.Key.Value] = stored; var homes = new List(); var globals = new Dictionary(_globals); var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); @@ -3218,23 +3226,57 @@ namespace GameCult.Caching void Claim(CultStoredDocument stored, bool upsert) { - var home = Home(stored.Descriptor.DocumentType); - if (home == null) + if (WriteRefusal(stored, upsert, batch, out var home) is { } refusal) throw refusal; + if (home != null) homes.Add(home); + } + } + + // The one answer to whether a record may be written (upsert: a write of its type) or its key removed, before anything is staged. + // Claim throws it; IsWritable asks it of the held record's removal form. Null when the write may land; home is where it lands, + // null for a store-less cache. batch holds the records the same batch admits, by key. + private Exception? WriteRefusal(CultStoredDocument stored, bool upsert, IReadOnlyDictionary? batch, out CacheBackingStore? home) + { + home = Home(stored.Descriptor.DocumentType); + if (home == null) + { + if (_stores.Count > 0) + return new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}."); + // No store, no file to judge, but the same rule: a record is replaced only by its own type's write. + return upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor) + ? CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor) + : null; + } + + if (home.IsReadOnly) + return new InvalidOperationException( + $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); + return home.Refusal(stored.Key.Value, upsert ? stored.Descriptor : null) + ?? (upsert && stored.Flattens ? FlattenRefusal(stored, batch) : null); + } + + // A flatten writes its base chain's decoded view into a plain record, so it is refused when a record on that chain is not owned + // by its home store (the slots or drift that record's decode dropped would be lost): that ancestor's own refusal. The chain starts + // at the staged delta's base when the variant is flattened as it lands, else at the held variant's; a key the batch admits is + // judged by its own claim, and the walk follows its staged form. + private CultSchemaConflictException? FlattenRefusal(CultStoredDocument flatten, IReadOnlyDictionary? batch) + { + var delta = flatten.Variant ?? (_entries.TryGetValue(flatten.Key.Value, out var variant) ? variant.Variant : null); + var walked = new HashSet(StringComparer.Ordinal) { flatten.Key.Value }; + for (var key = delta?.BaseKey; key != null && walked.Add(key);) + { + CultStoredDocument? ancestor; + if (batch == null || !batch.TryGetValue(key, out ancestor)) { - if (_stores.Count > 0) - throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}."); - // No store, no file to judge, but the same rule: a record is replaced only by its own type's write. - if (upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor)) - throw CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor); - return; + if (!_entries.TryGetValue(key, out ancestor)) + return null; + if (Home(ancestor.Descriptor.DocumentType)?.Refusal(key, null) is { } refusal) + return refusal; } - if (home.IsReadOnly) - throw new InvalidOperationException( - $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); - home.RefuseUndeclared(stored.Key.Value, upsert ? stored.Descriptor : null); - homes.Add(home); + key = ancestor.Variant?.BaseKey; } + + return null; } private sealed class VariantPlan -- 2.47.3 From 63911a8db058a28bdf7c4f8e56c776dbbb51d712 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 00:33:08 +0000 Subject: [PATCH 11/20] Contracts, CHANGELOG, vector README: written-count slots, consent cap, flatten refusal, IsWritable is the write's answer CHANGELOG Unreleased (org.gamecult.cultlib): Breaking gains the flatten-through-an-unowned-base refusal, MintElementIds refusing a store holding a marked memory-only record, and the written-count rule with the [MessagePackFormatter] and unmeasurable-type exclusions; Added corrects IsWritable. The vector README says what a runtime walks for extra-slot-consent.cc, the unit-arm record and slot-outcomes.msgpack. Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 10 +++- .../cultcache-schema-compatibility.md | 19 +++++++- tests/vectors/union-parity/README.md | 48 +++++++++++++++---- unity/org.gamecult.cultlib/CHANGELOG.md | 5 +- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 851d261e..65e3a5af 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -205,8 +205,14 @@ the next write stores it under the writing type's own id and sheds the slots. A record whose payload carries a slot past the slots its reader declares, at any depth (nested object, list element, union arm, unit arm), is read-only the same way, whatever id it is stored under. The one consent is the declaration above, and it reaches only depth-0 slots the schema comparison reports ignored under a -listed compatible id: a mark anywhere else, or at depth 0 under the reader's own id, is not consented to. -`CultCache.IsWritable(key)` answers the same question the staging refusal asks. +listed compatible id, up to the highest slot it reports: a mark anywhere else, past that slot, or at depth 0 +under the reader's own id, is not consented to. Declared slots are what the type's own formatter writes; a +type with its own `[MessagePackFormatter]`, or one no instance of which can be written, is not checked. +A flatten of a variant whose base chain holds a record that is not owned is refused with that record's +refusal, because it would write the base's decoded view; the variant's own writes persist only its delta and +stay allowed. A plain write to another key of a value read from a read-only record is a copy this rule does +not police. `CultCache.IsWritable(key)` answers what the staging refusal answers for a removal or a write of +the record's type, read-only and missing home stores included; it does not predict that flatten. The rule is checked again where the write lands, under the store's lock: a write or removal at a key whose durable record is not owned, because another writer rewrote it after this cache diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index c638050e..894865c9 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -119,8 +119,23 @@ served, `CultStoredDocument.UnreadSlots` names the first type read with extra sl `CultSchemaConflictException` (`UnreadSlots` set; the message names the type and the counts, never a slot value). A missing trailing slot keeps its default and stays writable; a non-nil value in a retired gap below the highest key is not extra. Declaring an id compatible consents only to depth-0 slots the schema comparison -reports ignored under that id; any other mark (nested, list element, arm, unit arm, or depth 0 under the -reader's own id) keeps the record read-only whatever is declared. +reports ignored under that id, and only up to the highest slot it reports: a payload carrying a slot past that +one is not consented to. Any other mark (nested, list element, arm, unit arm, or depth 0 under the reader's +own id) keeps the record read-only whatever is declared. + +A type's declared slots are the array its own formatter writes for one instance (an uninitialized instance, +else one from a parameterless constructor), never a reflected member list, so a non-public `[Key]` member of +an `AllowPrivate` type counts. Only concrete int-keyed `[MessagePackObject]` types are checked. A type that +names its own `[MessagePackFormatter]` owns its layout and its tolerance and is not checked, and neither is a +type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an +uninitialized one): extra slots in such a type are read past and dropped on the next write, as before. + +`Flatten` of a variant writes its base chain's decoded view into a plain record, so it is refused when any +record on that chain is not owned by its home store (a mark nothing explains, or undeclared drift that drops +slots), with that record's own refusal naming its key. Writing or removing the variant itself stays allowed: +it persists only its delta. `CultCache.IsWritable(key)` is the write's own answer for a removal or a write of +the record's type: it is also false when the record's home store is read-only or no attached store is home to +its type. The one write it does not predict is that flatten. The cache emits a typed migration report with: diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index 618562f1..9ecf47f5 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -55,7 +55,9 @@ string `"extra-slot"` appended as one more slot of one object; a reader decodes from each. The fifth is a bare arm of union `UnitOutcome` (0 Waiting `[Message]`; 2 Denied2, -2147483648 Floor2 and 2147483647 Ceiling2, each `[]`), the union of the `unit-arm-*` files; a reader decodes Denied2 from it. -An object's declared slots are its highest key plus one. Only an array longer than that marks: a short array +An object's declared slots are the slots its type's own formatter writes for one instance (for these fixtures, +the highest key plus one); a reader derives them from the layout it writes, never from a list of public members. +Only an array longer than that marks: a short array (missing trailing slots) keeps the defaults and is writable, and a non-nil value in a retired slot below the highest key (Image's slot 2) does not mark. A union adds no depth: an arm's array is counted against the arm's own slots. @@ -81,21 +83,47 @@ Load it as a store record under the reader's own schema id (the C# reference put 2. the record reports the mark as in the table: the first type read with extra slots, the slots it carried and the slots the reader declares, and the count of objects that carried extra slots (1 here); 3. the record is not writable (C#: `CultCache.IsWritable(key)` false), and a key no store holds is writable; -4. a replacing write, a variant write and a removal of the key (and a flatten, when the marked record is a variant) - are refused before anything is staged, +4. a replacing write, a variant write and a removal of the key, and a flatten of a variant whose base chain holds + the marked record, are refused before anything is staged, with the runtime's schema conflict refusal carrying the mark; its message names the type and both counts and never contains `extra-slot`, the slot's value; 5. a write to another key of the same store succeeds. Consent: a mark is explained only when it is at depth 0 and the schema comparison of the record's stored id against -the reader reported ignored extra slots, i.e. the record is stored under an id the reader lists compatible and the -catalog entry for that id has more members. Declaring that id consents to an explained mark only. Every mark in the -table is unexplained (stored under the reader's own id), so each record stays read-only whatever is declared. +the reader reported ignored extra slots covering it: the record is stored under an id the reader lists compatible, +that id's catalog entry has more members, and the mark's slot count is at most the highest reported slot plus one. +Declaring that id consents to an explained mark only. Every mark in the table above is unexplained (stored under +the reader's own id), so each record stays read-only whatever is declared. -`extra-slot-store.cc` is a single-file store of two `tests.union_surface` records, the store's clock pinned to -2026-01-01T00:00:00Z as `union-surface.cc`'s: `marked`, whose payload is `extra-slot-nested-record.msgpack`, and -`clean`, the value of `union-surface.msgpack`. Open it, assert `marked` read-only and `clean` writable, write `clean` -and flush: `marked`'s payload bytes, schema id and stored-at, and every catalog entry's bytes, equal the originals. +## Slot stores (`slot-outcomes.msgpack`) + +Two single-file stores, each record written as a value and its payload then replaced, the store's clock pinned to +2026-01-01T00:00:00Z as `union-surface.cc`'s: + +- `extra-slot-store.cc`, read with `tests.union_surface` (schema `tests.union_surface.v1`) and `tests.slot_unit` + (schema `tests.slot_unit.v1`, a document `[Name, Outcome: UnitOutcome]`). `marked` is + `extra-slot-nested-record.msgpack`; `clean` is `union-surface.msgpack`; `unit-arm` is the `tests.slot_unit` + document `["unit-arm", [2, ["extra-slot"]]]`, whose Denied2 arm sits at depth 1 (the document is depth 0; a union + adds none). +- `extra-slot-consent.cc`: every record is stored under `tests.union_surface.wide`, whose catalog entry is + `tests.union_surface`'s seven members plus slot 7 (`Wider`). Read it with a `tests.union_surface` type of schema + `tests.union_surface.v2` (the reference names it `SurfaceDeclaring`) that lists `tests.union_surface.wide` + compatible, so the comparison reports slot 7 ignored. `consented` is `union-surface.msgpack` plus one root slot + (slot 7); `over-consent` plus two (slots 7 and 8, past the highest reported); `nested` is + `extra-slot-nested-record.msgpack` plus one root slot; `clean` is `union-surface.msgpack`. + +`slot-outcomes.msgpack` is a second outcomes file, as `bounds-outcomes.msgpack` is, so readers of `outcomes.msgpack` +stay green. It is an array of rows `[store, key, writable, explained, type, slots, declared, occurrences]`, one +per record of both stores: `writable` is what `IsWritable` answers; for an unmarked record the last five are nil, +otherwise they are the mark: whether it is explained, the first type read with extra slots, the slots it carried, +the slots the reader declares, and how many objects carried extra slots. A depth-0 type is the reader's document +type (the reference's names: `SurfaceDeclaring` for the consent store). For each row, open a copy of the store, +assert the mark and `IsWritable`, then remove the key: refused (with the mark, the file unchanged) exactly where +`writable` is false. In the consent store a write of `consented` lands and a write of `over-consent` is refused. + +The carried-verbatim check: open `extra-slot-store.cc`, assert `marked` read-only and `clean` writable, write +`clean` and flush: `marked`'s payload bytes, schema id and stored-at, and every catalog entry's bytes, equal the +originals. ## Bounds (`bounds-outcomes.msgpack`) diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index c2fc2a1f..49fcc5ec 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -8,10 +8,13 @@ All notable changes to this package are documented in this file. - A single-file store holding two records under one key is unreadable: the reader refuses it with `CultStoreUnreadableException` naming the key (keys compare by ordinal code units, no case folding or normalisation), so a cache, `ReadStoreSnapshot` and a flush refuse the file and leave it as it is. Before, the later record silently replaced the earlier. A directory store reads its manifest through the same reader and refuses a duplicate key the same way. Repair the duplicate with the runtime that wrote it before reading the store here. - A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. +- `Flatten` (and `FlattenAsync`) of a variant whose base chain holds a record its store does not own (a record held read-only for extra slots, or undeclared drift that drops slots) is refused with `CultSchemaConflictException` naming that base record, and the store is left as it was. Before, the flatten wrote the base's decoded view into a plain record and the base's unread data was lost for that variant. Writing and removing the variant itself are still accepted: they persist only its delta. +- `MintElementIds` refuses a home store's commit while that store holds a record read-only for extra slots whose element ids exist only in memory, until the build that declares those slots rewrites it; commits to earlier stores stand. +- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. A type carrying `[MessagePackFormatter]` is not checked for extra slots (it owns its layout), and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance); extra slots in those are read past and dropped on the next write, as before. ### Added -- `CultCache.IsWritable(CultRecordKey)`: whether a write or removal of the key would be let through, answered by the same predicate the write uses (false for a foreign record, undeclared drift that drops slots, or unread slots; true for a key no store holds). Live and advisory: the write still decides. +- `CultCache.IsWritable(CultRecordKey)`: whether a removal of the key, or a write of the record's own type, would be let through, answered by the write's own decision: false for a foreign record, undeclared drift that drops slots, unread slots, a read-only home store, or a record no attached store is home to; true for a key the cache does not hold unless a store lists it foreign. It does not predict a `Flatten` refused for its base chain. Live and advisory: the write still decides. - `CultUnreadSlots` (`TypeName`, `SlotCount`, `DeclaredCount`, `Occurrences`, `Explained`), `CultStoredDocument.UnreadSlots`, and `CultSchemaConflictException.UnreadSlots` with its constructor overload: what a load found past the reader's slots, holding no slot value. The refusal message names the type and both counts, never a value. ## [3.0.0] - 2026-11-11 -- 2.47.3 From 080054bf9974daa61bd314785bd3c0d04bd3e8b3 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:13:44 +0000 Subject: [PATCH 12/20] Slot tests, red: a resolver-formatted type and a [MessagePackObject] collection are not measured; flatten-chain vector row ResolvedBag (an assembly CultCacheFormatterResolver writes [count, items...]) and TagList (List with [MessagePackObject]) are falsely marked by the attribute-based guard (Soul s2, guard-asks-attribute-not-formatter). extra-slot-store.cc gains marked-variant, a variant of marked; slot-outcomes.msgpack rows gain a ninth column, the key whose refusal refuses a Flatten. Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/AssemblyInfo.cs | 1 + .../SlotReadOnlyTests.cs | 50 ++++++++++++++++-- .../UnionParityVectorTests.cs | 23 +++++--- .../vectors/union-parity/extra-slot-store.cc | Bin 3716 -> 3873 bytes .../union-parity/slot-outcomes.msgpack | Bin 322 -> 378 bytes 5 files changed, 64 insertions(+), 10 deletions(-) diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 708f0a09..85ae1449 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -10,6 +10,7 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))] +[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.ResolvedBagResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index daa4beba..3f3bcd15 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -160,10 +160,10 @@ namespace GameCult.Caching.Tests { var reader = new MessagePackReader(VectorFiles.Committed("union-parity", "slot-outcomes.msgpack")); var rows = reader.ReadArrayHeader(); - Assert.That(rows, Is.EqualTo(7)); + Assert.That(rows, Is.EqualTo(8)); for (var i = 0; i < rows; i++) { - Assert.That(reader.ReadArrayHeader(), Is.EqualTo(8)); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(9)); var store = reader.ReadString()!; var key = new CultRecordKey(reader.ReadString()!); var writable = reader.ReadBoolean(); @@ -172,6 +172,7 @@ namespace GameCult.Caching.Tests var counts = new int?[3]; for (var c = 0; c < 3; c++) counts[c] = reader.TryReadNil() ? null : reader.ReadInt32(); var (slots, declared, occurrences) = (counts[0], counts[1], counts[2]); + var flattenRefusedBy = reader.TryReadNil() ? null : reader.ReadString(); var row = $"{store} {key.Value}"; var path = Path.Combine(_directory, $"{i}-{store}"); @@ -192,6 +193,13 @@ namespace GameCult.Caching.Tests Assert.That(cache.IsWritable(key), Is.EqualTo(writable), row); Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable"); + if (flattenRefusedBy != null) + { + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Flatten(key)), $"{row}: flatten")!; + Assert.That(refusal.RecordKey, Is.EqualTo(flattenRefusedBy), $"{row}: the refusal is the base's own"); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused flatten leaves the store as it was"); + } + if (writable) { Assert.That(cache.Commit(batch => batch.Remove(key)), Is.True, row); @@ -305,7 +313,6 @@ namespace GameCult.Caching.Tests reader.AddBackingStore(Store()); var unread = Stored(reader, Marked).UnreadSlots; Assert.That(unread, Is.Not.Null, "the store kind decodes through the marking path"); - Assert.That(unread!.Explained, Is.True, "depth 0 and reported by the catalog comparison"); Assert.That(unread.TypeName, Is.EqualTo(nameof(NarrowDoc))); Assert.That(unread.SlotCount, Is.EqualTo(3)); Assert.That(unread.DeclaredCount, Is.EqualTo(2)); @@ -369,6 +376,37 @@ namespace GameCult.Caching.Tests } } + // A [MessagePackObject] type an assembly resolver formats as [count, items...], with no [MessagePackFormatter] attribute: the + // resolver owns its layout. + [MessagePackObject] + public sealed class ResolvedBag + { + [Key(0)] public List Items = new(); + } + + public sealed class ResolvedBagResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ResolvedBagResolver(); + private static readonly IMessagePackFormatter Formatter = new ResolvedBagFormatter(); + + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(ResolvedBag) ? (IMessagePackFormatter)Formatter : null; + + private sealed class ResolvedBagFormatter : IMessagePackFormatter + { + private static readonly BagFormatter Bag = new(); + + public void Serialize(ref MessagePackWriter writer, ResolvedBag? value, MessagePackSerializerOptions options) => + Bag.Serialize(ref writer, value == null ? null : new Bag { Items = value.Items }, options); + + public ResolvedBag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => + Bag.Deserialize(ref reader, options) is { } bag ? new ResolvedBag { Items = bag.Items } : null; + } + } + + // A [MessagePackObject] collection: MessagePack writes it as its items, not by [Key]. + [MessagePackObject] + public class TagList : List { } + [CultDocument("tests.slot_measured", "tests.slot_measured.v1")] [MessagePackObject] public sealed class MeasuredDoc @@ -378,6 +416,8 @@ namespace GameCult.Caching.Tests [Key(2)] public Counted? Counted; [Key(3)] public Uncounted? Uncounted; [Key(4)] public Bag? Bag; + [Key(5)] public ResolvedBag? Resolved; + [Key(6)] public TagList? Tags; } private static readonly CultDocumentRegistry MeasuredRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(MeasuredDoc) }); @@ -388,6 +428,8 @@ namespace GameCult.Caching.Tests "constructed" => new MeasuredDoc { Name = member, Counted = new Counted { Count = 2 } }, "unmeasurable" => new MeasuredDoc { Name = member, Uncounted = new Uncounted(2) }, "own-formatter" => new MeasuredDoc { Name = member, Bag = new Bag { Items = { "x", "y", "z" } } }, + "resolver-formatted" => new MeasuredDoc { Name = member, Resolved = new ResolvedBag { Items = { "x", "y", "z" } } }, + "collection" => new MeasuredDoc { Name = member, Tags = new TagList { "a", "b", "c" } }, _ => throw new ArgumentOutOfRangeException(nameof(member), member, null) }; @@ -395,6 +437,8 @@ namespace GameCult.Caching.Tests [TestCase("constructed", TestName = "ATypeMeasuredThroughItsConstructorDoesNotMark")] [TestCase("unmeasurable", TestName = "AnUnmeasurableTypeIsNotGuarded")] [TestCase("own-formatter", TestName = "ATypeWithItsOwnFormatterIsNotMeasuredByKeys")] + [TestCase("resolver-formatted", TestName = "ATypeAResolverFormatsIsNotMeasuredByKeys")] + [TestCase("collection", TestName = "AMessagePackObjectCollectionIsNotMeasuredByKeys")] public void ARecordWrittenByItsOwnTypeDoesNotMark(string member) { var path = Path.Combine(_directory, "measured.cc"); diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 30e8f744..067a6cef 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -378,14 +378,16 @@ namespace GameCult.Caching.Tests _ => throw new ArgumentOutOfRangeException(nameof(store), store, null) }; - // One row of slot-outcomes.msgpack: what the store's reader holds at the key. The mark fields are null for an unmarked record. - private sealed record SlotOutcome(string Store, string Key, bool Writable, bool? Explained, string? Type, int? Slots, int? Declared, int? Occurrences); + // One row of slot-outcomes.msgpack: what the store's reader holds at the key. The mark fields are null for an unmarked record; + // FlattenRefusedBy is the key whose refusal refuses a Flatten of the record, null when the record is no variant or flattens. + private sealed record SlotOutcome(string Store, string Key, bool Writable, bool? Explained, string? Type, int? Slots, int? Declared, int? Occurrences, string? FlattenRefusedBy = null); private static readonly SlotOutcome[] SlotOutcomes = { new("extra-slot-store.cc", "marked", false, false, nameof(Composition), 3, 2, 1), new("extra-slot-store.cc", "clean", true, null, null, null, null, null), new("extra-slot-store.cc", "unit-arm", false, false, nameof(Denied2), 1, 0, 1), + new("extra-slot-store.cc", "marked-variant", true, null, null, null, null, null, "marked"), new("extra-slot-consent.cc", "consented", true, true, nameof(SurfaceDeclaring), 8, 7, 1), new("extra-slot-consent.cc", "over-consent", false, false, nameof(SurfaceDeclaring), 9, 7, 1), new("extra-slot-consent.cc", "nested", false, false, nameof(SurfaceDeclaring), 8, 7, 2), @@ -397,12 +399,13 @@ namespace GameCult.Caching.Tests w.WriteArrayHeader(SlotOutcomes.Length); foreach (var row in SlotOutcomes) { - w.WriteArrayHeader(8); + w.WriteArrayHeader(9); w.Write(row.Store); w.Write(row.Key); w.Write(row.Writable); if (row.Explained is { } explained) w.Write(explained); else w.WriteNil(); if (row.Type is { } type) w.Write(type); else w.WriteNil(); foreach (var count in new[] { row.Slots, row.Declared, row.Occurrences }) if (count is { } value) w.Write(value); else w.WriteNil(); + if (row.FlattenRefusedBy is { } refusedBy) w.Write(refusedBy); else w.WriteNil(); } }); @@ -482,8 +485,10 @@ namespace GameCult.Caching.Tests } // A single-file store of the records, each written as its type's value with the store's clock pinned as union-surface.cc's, then - // its payload replaced by bytes no current writer produces; edit changes the snapshot further before it is serialized. - private static byte[] SlotStore(CultDocumentRegistry registry, (string Key, object Value, byte[] Payload)[] records, Action? edit = null) + // its payload replaced by bytes no current writer produces; each variant is written over its base (overriding the name to its + // key) before the payloads are replaced; edit changes the snapshot further before it is serialized. + private static byte[] SlotStore(CultDocumentRegistry registry, (string Key, object Value, byte[] Payload)[] records, Action? edit = null, + params (string Key, string Base)[] variants) { var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}"); Directory.CreateDirectory(work); @@ -495,6 +500,9 @@ namespace GameCult.Caching.Tests cache.Clock = () => StoredAt; foreach (var (key, value, _) in records) cache.UpsertAsync(value.GetType(), value, new CultRecordKey(key)).GetAwaiter().GetResult(); + foreach (var (key, baseKey) in variants) + Assert.That(cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), new CultRecordKey(baseKey), + new[] { cache.Override(typeof(UnionSurface), nameof(UnionSurface.Id), key) })), Is.True); cache.FlushAsync().GetAwaiter().GetResult(); } @@ -511,7 +519,8 @@ namespace GameCult.Caching.Tests } // The store the slot-readonly tests open and the other runtimes read: "marked" holds the extra-slot-nested-record payload, - // "clean" the ordinary value, and "unit-arm" a tests.slot_unit document ["unit-arm", [2, ["extra-slot"]]]. + // "clean" the ordinary value, "unit-arm" a tests.slot_unit document ["unit-arm", [2, ["extra-slot"]]], and "marked-variant" a + // variant of "marked" overriding only its Id: writable, but a flatten of it is refused for its base. [Test] public void ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord() { @@ -520,7 +529,7 @@ namespace GameCult.Caching.Tests ("marked", Constructed(), Committed("extra-slot-nested-record.msgpack")), ("clean", Constructed(), Surface()), ("unit-arm", new UnitDoc { Name = "unit-arm", Outcome = new Denied2() }, Raw(Slots(Str("unit-arm"), ArmOf(2, ExtraSlot)))), - })); + }, null, ("marked-variant", "marked"))); } // The consent store: every record under the wide id, whose catalog entry is UnionSurface's 7 members plus slot 7. Its reader diff --git a/tests/vectors/union-parity/extra-slot-store.cc b/tests/vectors/union-parity/extra-slot-store.cc index 524479241d2caeb3f15bf3aa8404e3bbc9f1249a..c5bc6cfb6a8a09bfdbfd850a8467d000b8a0dde6 100644 GIT binary patch delta 92 zcmZpXT`0HVA{WP$DNB=cQWNtg3o?67UdN-l*`0eWlfcw5la@iGCQf0PIEi7w0(6yMcqae=c9A5c delta 33 pcmZ1|*CM;&A{WQxDNB=cQWNtg3o?67UdN-l*`0eW)8vVK69C>544VJ| diff --git a/tests/vectors/union-parity/slot-outcomes.msgpack b/tests/vectors/union-parity/slot-outcomes.msgpack index eed3290487d9068da5b702999bf0ce790bd099de..cd9a9ef87c49baebd83bc9a95545d34efe129288 100644 GIT binary patch delta 110 zcmX@a^oxmU#>|NUA``_Vm=1s#d`pvaQWNtI9{>ZOh!#X#c3o~_QFdyIZdqbcW@26m iM8PtM;LOPijA9cfDgc#&StpdCtjSD_vJj>c;{gCn%PjK% delta 57 ycmeyxbcl&*`izMNA``_V7=g3`h!B~WCI+O}n}Gl6L} -- 2.47.3 From 6d4421fb659e5d3214ca734c249880ff03b3f12d Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:14:20 +0000 Subject: [PATCH 13/20] CultUnionGuard: measure a type only when the chain hands out MessagePack's own [Key]-layout formatter Guarded decided from attributes, then measured whatever formatter the composite handed out, so a consumer resolver's [count, items...] type and a [MessagePackObject] collection were falsely marked (Soul s2 guard-asks-attribute-not-formatter). KeyLayout decides from the formatter itself, by identity with StandardResolver's (or StandardResolverAllowPrivate's), excluding the attribute and generic-collection resolvers' instances, as ResolverClaim does. The [MessagePackFormatter] attribute check is deleted: that door is now the AttributeFormatterResolver identity. Co-Authored-By: Claude Opus 5.5 --- .../CultUnionGuard.cs | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs index 1a2a535e..b8e38f3c 100644 --- a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs +++ b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using MessagePack; using MessagePack.Formatters; +using MessagePack.Resolvers; namespace GameCult.Caching.MessagePack; @@ -21,16 +22,35 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol { var arms = typeof(T).GetCustomAttributes(inherit: false).ToArray(); if (arms.Length == 0) - return Guarded(typeof(T)) && inner.GetFormatter() is { } objectFormatter ? new SlotFormatter(objectFormatter) : null; + return inner.GetFormatter() is { } objectFormatter && KeyLayout(objectFormatter) ? new SlotFormatter(objectFormatter) : null; return inner.GetFormatter() is not { } formatter ? null : new Formatter(formatter, arms); } - // A concrete int-keyed [MessagePackObject] type whose layout MessagePack derives from its [Key] members. A type that names its own - // [MessagePackFormatter] owns its layout and its tolerance (it may read a legacy shape or skip fields on purpose), so it is not guarded. - private static bool Guarded(Type type) => - !type.IsAbstract && !type.IsInterface && !type.IsGenericTypeDefinition && - type.GetCustomAttribute(inherit: true) is { KeyAsPropertyName: false } && - type.GetCustomAttribute(inherit: true) == null; + // The one decision of whether a type is measured: the formatter the chain hands out for an int-keyed [MessagePackObject] type is + // MessagePack's own [Key]-layout formatter, the very instance StandardResolver (or its AllowPrivate twin) builds for it. A formatter a + // consumer resolver or a [MessagePackFormatter] attribute supplies owns its layout and its tolerance, and a [MessagePackObject] + // collection is written as its items by the generic resolver: none of those is measured. Identity, as ResolverClaim asks it. + private static bool KeyLayout(IMessagePackFormatter formatter) + { + if (typeof(T).IsAbstract || typeof(T).IsInterface || + typeof(T).GetCustomAttribute(inherit: true) is not { KeyAsPropertyName: false }) + return false; + if (ReferenceEquals(formatter, Of(AttributeFormatterResolver.Instance)) || ReferenceEquals(formatter, Of(DynamicGenericResolver.Instance))) + return false; + return ReferenceEquals(formatter, Of(StandardResolver.Instance)) || ReferenceEquals(formatter, Of(StandardResolverAllowPrivate.Instance)); + } + + private static IMessagePackFormatter? Of(IFormatterResolver resolver) + { + try + { + return resolver.GetFormatter(); + } + catch (Exception) + { + return null; + } + } // Counts a concrete object's array header against its declared slots and notes the excess into the open CultSlotScope, then lets the // inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through. -- 2.47.3 From c452b028fd77f56a4a9ecd62b60b4f63ab594ec3 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:14:20 +0000 Subject: [PATCH 14/20] CultCache.IsWritable: an unheld key is answered by the store's own removal check, not a copy of its foreign clause Soul s2 iswritable-unheld-copies-foreign. The copy was induced by r5's negative grep forbidding Store.Refusal(key.Value, null) anywhere in IsWritable; the held branch still asks WriteRefusal. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index b386be0c..48bad3a3 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2669,8 +2669,8 @@ namespace GameCult.Caching /// Whether a removal of the key, or a write of the record's own type, would be let through: the write's own decision, asked of /// the record the cache holds. False when no attached store is home to the record's type, when its home store is read-only, or /// when that store holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's, - /// see ). For a key the cache does not hold, false only when a store lists it - /// foreign. One write it does not predict: a Flatten of a variant whose base chain holds an unowned record is refused, though + /// see ). For a key the cache does not hold, the removal check every store makes of it + /// (false when a store lists it foreign). One write it does not predict: a Flatten of a variant whose base chain holds an unowned record is refused, though /// the variant itself is writable. Live and advisory: the write still decides. /// public bool IsWritable(CultRecordKey key) @@ -2678,7 +2678,7 @@ namespace GameCult.Caching lock (Gate()) return _entries.TryGetValue(key.Value, out var held) ? WriteRefusal(held, upsert: false, null, out _) == null - : _stores.All(entry => entry.Store.ForeignRecords.All(record => !string.Equals(record.Key, key.Value, StringComparison.Ordinal))); + : _stores.All(entry => entry.Store.Refusal(key.Value, null) == null); } public object? Get(CultRecordKey key) -- 2.47.3 From 5e771602904dd048c599aa3fcda47ccedd85b6d7 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:14:48 +0000 Subject: [PATCH 15/20] Contracts, CHANGELOG, vector README: which types are exempt from measuring, the ruled gap, the user code the measure runs Exempt: any type not written by MessagePack's [Key]-layout formatter (own [MessagePackFormatter], a consumer resolver, a [MessagePackObject] collection) and a type no instance of which can be written (cultlib-gaps:ruling:slot-unmeasurable-type). The measure runs [Key] getters and a parameterless constructor once per type per process; a failure leaves the type unchecked. The README states the leaf exemptions and the flattenRefusedBy column. Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 6 +++-- .../cultcache-schema-compatibility.md | 22 ++++++++++++++----- tests/vectors/union-parity/README.md | 19 +++++++++++----- unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 65e3a5af..5ac6c3b5 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,8 +206,10 @@ A record whose payload carries a slot past the slots its reader declares, at any element, union arm, unit arm), is read-only the same way, whatever id it is stored under. The one consent is the declaration above, and it reaches only depth-0 slots the schema comparison reports ignored under a listed compatible id, up to the highest slot it reports: a mark anywhere else, past that slot, or at depth 0 -under the reader's own id, is not consented to. Declared slots are what the type's own formatter writes; a -type with its own `[MessagePackFormatter]`, or one no instance of which can be written, is not checked. +under the reader's own id, is not consented to. Declared slots are what the type's own formatter writes; only +a type MessagePack writes by its `[Key]` layout is checked, and one no instance of which can be written is not +(`cultlib-gaps:ruling:slot-unmeasurable-type`; see the schema-compatibility contract for both exemptions and +for the user code the measure runs). A flatten of a variant whose base chain holds a record that is not owned is refused with that record's refusal, because it would write the base's decoded view; the variant's own writes persist only its delta and stay allowed. A plain write to another key of a value read from a read-only record is a copy this rule does diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index 894865c9..c99d6292 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -123,12 +123,22 @@ reports ignored under that id, and only up to the highest slot it reports: a pay one is not consented to. Any other mark (nested, list element, arm, unit arm, or depth 0 under the reader's own id) keeps the record read-only whatever is declared. -A type's declared slots are the array its own formatter writes for one instance (an uninitialized instance, -else one from a parameterless constructor), never a reflected member list, so a non-public `[Key]` member of -an `AllowPrivate` type counts. Only concrete int-keyed `[MessagePackObject]` types are checked. A type that -names its own `[MessagePackFormatter]` owns its layout and its tolerance and is not checked, and neither is a -type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an -uninitialized one): extra slots in such a type are read past and dropped on the next write, as before. +A type's declared slots are the array its own formatter writes for one instance, never a reflected member +list, so a non-public `[Key]` member of an `AllowPrivate` type counts. Only a concrete int-keyed +`[MessagePackObject]` type whose formatter is MessagePack's own `[Key]`-layout formatter is checked. Not +checked, and so read past with any extra slots dropped on the next write, as before: + +- a type formatted by anything else: its own `[MessagePackFormatter]`, a `CultCacheFormatterResolver` (or any + resolver ahead of MessagePack's), or the collection formatter of a `[MessagePackObject]` collection type + such as a `List` subclass. Such a formatter owns its layout and its tolerance; +- a type no instance of which can be written: no write of an uninitialized instance and no write of one + from a parameterless constructor succeeds (for example a `[Key]` getter that throws on an uninitialized + instance and no parameterless constructor). This gap is ruled: `cultlib-gaps:ruling:slot-unmeasurable-type`. + +Measuring runs user code once per type per process, on the first read of the type in a store load: the +type's `[Key]` getters on an uninitialized instance and, if that write throws, its parameterless +constructor (public or not) and the getters on that instance. Any failure of both leaves the type unchecked +for the rest of the process, and nothing else: it does not fail the read. `Flatten` of a variant writes its base chain's decoded view into a plain record, so it is refused when any record on that chain is not owned by its home store (a mark nothing explains, or undeclared drift that drops diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index 9ecf47f5..94597ed2 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -57,6 +57,12 @@ from each. The fifth is a bare arm of union `UnitOutcome` (0 Waiting `[Message]` An object's declared slots are the slots its type's own formatter writes for one instance (for these fixtures, the highest key plus one); a reader derives them from the layout it writes, never from a list of public members. +Only a type the runtime writes by its own positional `[Key]` layout is checked. Exempt, as in the C# reference +(`cultlib-gaps:ruling:leaf-acceptance-follows-reference`): a type the runtime formats some other way (a +user codec or formatter, a hand-written serializer, a resolver ahead of the runtime's own, a collection type +written as its items), and a type no instance of which the runtime can write for measuring +(`cultlib-gaps:ruling:slot-unmeasurable-type`). Neither is ever marked. No vector carries an exempt type: each +runtime pins its own exemption with its own fixture. Only an array longer than that marks: a short array (missing trailing slots) keeps the defaults and is writable, and a non-nil value in a retired slot below the highest key (Image's slot 2) does not mark. A union adds no depth: an arm's array is counted against the arm's own slots. @@ -104,7 +110,7 @@ Two single-file stores, each record written as a value and its payload then repl (schema `tests.slot_unit.v1`, a document `[Name, Outcome: UnitOutcome]`). `marked` is `extra-slot-nested-record.msgpack`; `clean` is `union-surface.msgpack`; `unit-arm` is the `tests.slot_unit` document `["unit-arm", [2, ["extra-slot"]]]`, whose Denied2 arm sits at depth 1 (the document is depth 0; a union - adds none). + adds none); `marked-variant` is a variant of `marked` overriding only `Id` (slot 0) to `"marked-variant"`. - `extra-slot-consent.cc`: every record is stored under `tests.union_surface.wide`, whose catalog entry is `tests.union_surface`'s seven members plus slot 7 (`Wider`). Read it with a `tests.union_surface` type of schema `tests.union_surface.v2` (the reference names it `SurfaceDeclaring`) that lists `tests.union_surface.wide` @@ -113,12 +119,15 @@ Two single-file stores, each record written as a value and its payload then repl `extra-slot-nested-record.msgpack` plus one root slot; `clean` is `union-surface.msgpack`. `slot-outcomes.msgpack` is a second outcomes file, as `bounds-outcomes.msgpack` is, so readers of `outcomes.msgpack` -stay green. It is an array of rows `[store, key, writable, explained, type, slots, declared, occurrences]`, one -per record of both stores: `writable` is what `IsWritable` answers; for an unmarked record the last five are nil, -otherwise they are the mark: whether it is explained, the first type read with extra slots, the slots it carried, +stay green. It is an array of rows `[store, key, writable, explained, type, slots, declared, occurrences, flattenRefusedBy]`, +one per record of both stores: `writable` is what `IsWritable` answers; `flattenRefusedBy` is nil unless the record +is a variant whose flatten is refused, and then it is the key of the base-chain record whose own refusal refuses +it (the refusal names that key; the store is unchanged). For an unmarked record `explained` through `occurrences` +are nil, otherwise they are the mark: whether it is explained, the first type read with extra slots, the slots it carried, the slots the reader declares, and how many objects carried extra slots. A depth-0 type is the reader's document type (the reference's names: `SurfaceDeclaring` for the consent store). For each row, open a copy of the store, -assert the mark and `IsWritable`, then remove the key: refused (with the mark, the file unchanged) exactly where +assert the mark and `IsWritable`, flatten the key when `flattenRefusedBy` is set and assert the refusal, then +remove the key: refused (with the mark, the file unchanged) exactly where `writable` is false. In the consent store a write of `consented` lands and a write of `over-consent` is refused. The carried-verbatim check: open `extra-slot-store.cc`, assert `marked` read-only and `clean` writable, write diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 49fcc5ec..765d910a 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this package are documented in this file. - A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. - `Flatten` (and `FlattenAsync`) of a variant whose base chain holds a record its store does not own (a record held read-only for extra slots, or undeclared drift that drops slots) is refused with `CultSchemaConflictException` naming that base record, and the store is left as it was. Before, the flatten wrote the base's decoded view into a plain record and the base's unread data was lost for that variant. Writing and removing the variant itself are still accepted: they persist only its delta. - `MintElementIds` refuses a home store's commit while that store holds a record read-only for extra slots whose element ids exist only in memory, until the build that declares those slots rewrites it; commits to earlier stores stand. -- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. A type carrying `[MessagePackFormatter]` is not checked for extra slots (it owns its layout), and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance); extra slots in those are read past and dropped on the next write, as before. +- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked, and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked. ### Added -- 2.47.3 From 6403add7105ea858852a446bc944590596577027 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:21:51 +0000 Subject: [PATCH 16/20] SlotReadOnlyTests: restore the Explained assertion lost with a debug line; an unheld key a store lists foreign is not writable The assertion went out with 0dfc437b. The new test kills mutant M13 (the unheld branch answering true), which survived without it. Co-Authored-By: Claude Opus 5.5 --- .../GameCult.Caching.Tests/SlotReadOnlyTests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index 3f3bcd15..3d0e650d 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -313,6 +313,7 @@ namespace GameCult.Caching.Tests reader.AddBackingStore(Store()); var unread = Stored(reader, Marked).UnreadSlots; Assert.That(unread, Is.Not.Null, "the store kind decodes through the marking path"); + Assert.That(unread!.Explained, Is.True, "depth 0 and reported by the catalog comparison"); Assert.That(unread.TypeName, Is.EqualTo(nameof(NarrowDoc))); Assert.That(unread.SlotCount, Is.EqualTo(3)); Assert.That(unread.DeclaredCount, Is.EqualTo(2)); @@ -580,6 +581,21 @@ namespace GameCult.Caching.Tests Assert.That(refusal.Message, Does.Contain("read-only backing store")); } + // A key the cache does not hold, but a store carries as a foreign record: the store's removal check refuses it. + [Test] + public void IsWritableIsFalseForAnUnheldKeyAStoreListsForeign() + { + var path = Path.Combine(_directory, "foreign.cc"); + using (var writer = Open(path, GapRegistry)) + Assert.That(writer.Commit(batch => batch.Upsert(new GapDoc { Name = "g" }, new CultRecordHandle(Marked))), Is.True); + + using var cache = Open(path, SurfaceRegistry); + Assert.That(cache.GetStored(Marked), Is.Null, "the cache does not hold a record no registered type reads"); + Assert.That(cache.IsWritable(Marked), Is.False); + AssertWriteAgrees(cache, Marked, () => cache.Commit(batch => batch.Remove(Marked))); + Assert.That(cache.IsWritable(Unheld), Is.True); + } + [Test] public void IsWritableIsFalseForARecordNoAttachedStoreIsHomeTo() { -- 2.47.3 From 61307d2060b49fea55c16827890fc11bd949f52b Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:51:03 +0000 Subject: [PATCH 17/20] GeneratorConsumer: check the generated formatter behind the guard's wrapper (CultUnionGuard.Inner) The check that each document type runs MessagePack's source-generated formatter saw the guard's SlotFormatter and exited 1 (Soul s3 generator-consumer-check-red). The guard owns the one unwrap, internal, visible to the consumer through InternalsVisibleTo; that also exposes the library's internal GeneratedMessagePackResolver, so the consumer suppresses CS0436 (its own is the type used). Co-Authored-By: Claude Opus 5.5 --- .../CultUnionGuard.cs | 17 +++++++++++++++-- .../GameCult.Caching.MessagePack.csproj | 1 + .../GameCult.Caching.GeneratorConsumer.csproj | 3 +++ .../Program.cs | 3 ++- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs index b8e38f3c..86d26624 100644 --- a/src/GameCult.Caching.MessagePack/CultUnionGuard.cs +++ b/src/GameCult.Caching.MessagePack/CultUnionGuard.cs @@ -26,6 +26,15 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol return inner.GetFormatter() is not { } formatter ? null : new Formatter(formatter, arms); } + // The formatter that writes and reads the bytes: the one a guard wrapper delegates to, else the formatter itself. A check that a + // generated formatter is the one in use asks this (tests/GameCult.Caching.GeneratorConsumer). + internal static object Inner(object formatter) => formatter is IWrapper wrapper ? wrapper.Inner : formatter; + + private interface IWrapper + { + object Inner { get; } + } + // The one decision of whether a type is measured: the formatter the chain hands out for an int-keyed [MessagePackObject] type is // MessagePack's own [Key]-layout formatter, the very instance StandardResolver (or its AllowPrivate twin) builds for it. A formatter a // consumer resolver or a [MessagePackFormatter] attribute supplies owns its layout and its tolerance, and a [MessagePackObject] @@ -56,8 +65,10 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol // inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through. // Declared slots are what the type's own formatter writes: the array header of one instance written through it, measured on the // first read inside a scope (not in GetFormatter, so a self-referential type never recurses into its own resolver cache) and kept. - private sealed class SlotFormatter(IMessagePackFormatter inner) : IMessagePackFormatter + private sealed class SlotFormatter(IMessagePackFormatter inner) : IMessagePackFormatter, IWrapper { + object IWrapper.Inner => inner; + private const int NotMeasured = -1; // A type no instance of which can be written is not guarded: no count can exceed this. @@ -117,8 +128,10 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol } } - private sealed class Formatter(IMessagePackFormatter inner, UnionAttribute[] arms) : IMessagePackFormatter + private sealed class Formatter(IMessagePackFormatter inner, UnionAttribute[] arms) : IMessagePackFormatter, IWrapper { + object IWrapper.Inner => inner; + private readonly HashSet keys = arms.Select(arm => arm.Key).ToHashSet(); private readonly HashSet types = arms.Select(arm => arm.SubType).ToHashSet(); diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index 6261ca3d..63cbf614 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -11,6 +11,7 @@ + diff --git a/tests/GameCult.Caching.GeneratorConsumer/GameCult.Caching.GeneratorConsumer.csproj b/tests/GameCult.Caching.GeneratorConsumer/GameCult.Caching.GeneratorConsumer.csproj index 6d443a4f..4231195a 100644 --- a/tests/GameCult.Caching.GeneratorConsumer/GameCult.Caching.GeneratorConsumer.csproj +++ b/tests/GameCult.Caching.GeneratorConsumer/GameCult.Caching.GeneratorConsumer.csproj @@ -4,6 +4,9 @@ net10.0 enable false + + $(NoWarn);CS0436 diff --git a/tests/GameCult.Caching.GeneratorConsumer/Program.cs b/tests/GameCult.Caching.GeneratorConsumer/Program.cs index 2511ed80..ab83330f 100644 --- a/tests/GameCult.Caching.GeneratorConsumer/Program.cs +++ b/tests/GameCult.Caching.GeneratorConsumer/Program.cs @@ -20,7 +20,8 @@ var documents = new (Type Type, object Value)[] foreach (var (type, _) in documents) { - var formatter = options.Resolver.GetFormatterDynamic(type)!.GetType(); + // The guard wraps a measured type's formatter; the generated formatter it delegates to is the one that runs. + var formatter = CultUnionGuard.Inner(options.Resolver.GetFormatterDynamic(type)!).GetType(); if (formatter.Assembly.IsDynamic || !(formatter.FullName ?? "").Contains("GeneratedMessagePackResolver")) return Fail($"{type.Name} is formatted by {formatter.FullName}, not a generated formatter"); } -- 2.47.3 From b4d9a38d3d89faf2fa0aeb12fb9cabc6fb899291 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 01:51:03 +0000 Subject: [PATCH 18/20] A resolver that hands out MessagePack's own [Key] formatter exempts nothing: contract, CHANGELOG, README, test Soul s3 exemption-bullet-overstates-resolver. AnExtraSlotPastTheWrittenCountStillMarks pins it: RelayResolver hands out StandardResolver's formatter for Relayed, and its extra slot marks 2/1. Co-Authored-By: Claude Opus 5.5 --- .../cultcache-schema-compatibility.md | 8 +++--- tests/GameCult.Caching.Tests/AssemblyInfo.cs | 1 + .../SlotReadOnlyTests.cs | 27 +++++++++++++++++++ tests/vectors/union-parity/README.md | 5 ++-- unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index c99d6292..269c2f41 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -128,9 +128,11 @@ list, so a non-public `[Key]` member of an `AllowPrivate` type counts. Only a co `[MessagePackObject]` type whose formatter is MessagePack's own `[Key]`-layout formatter is checked. Not checked, and so read past with any extra slots dropped on the next write, as before: -- a type formatted by anything else: its own `[MessagePackFormatter]`, a `CultCacheFormatterResolver` (or any - resolver ahead of MessagePack's), or the collection formatter of a `[MessagePackObject]` collection type - such as a `List` subclass. Such a formatter owns its layout and its tolerance; +- a type formatted by anything else: its own `[MessagePackFormatter]`, a formatter of its own that a + `CultCacheFormatterResolver` (or any resolver ahead of MessagePack's) hands out, or the collection formatter + of a `[MessagePackObject]` collection type such as a `List` subclass. Such a formatter owns its layout and + its tolerance. A resolver that hands out MessagePack's own `[Key]`-layout formatter (StandardResolver's, or + StandardResolverAllowPrivate's) does not exempt the type: it is checked and marks; - a type no instance of which can be written: no write of an uninitialized instance and no write of one from a parameterless constructor succeeds (for example a `[Key]` getter that throws on an uninitialized instance and no parameterless constructor). This gap is ruled: `cultlib-gaps:ruling:slot-unmeasurable-type`. diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 85ae1449..8d619b43 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -11,6 +11,7 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))] [assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.ResolvedBagResolver))] +[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.RelayResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs index 3d0e650d..eb8bd64b 100644 --- a/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs +++ b/tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs @@ -404,6 +404,21 @@ namespace GameCult.Caching.Tests } } + // A type an assembly resolver claims by handing out MessagePack's own [Key]-layout formatter: still measured. + [MessagePackObject] + public sealed class Relayed + { + [Key(0)] public string A = string.Empty; + } + + public sealed class RelayResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new RelayResolver(); + + public IMessagePackFormatter? GetFormatter() => + typeof(T) == typeof(Relayed) ? global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter() : null; + } + // A [MessagePackObject] collection: MessagePack writes it as its items, not by [Key]. [MessagePackObject] public class TagList : List { } @@ -419,6 +434,7 @@ namespace GameCult.Caching.Tests [Key(4)] public Bag? Bag; [Key(5)] public ResolvedBag? Resolved; [Key(6)] public TagList? Tags; + [Key(7)] public Relayed? Relayed; } private static readonly CultDocumentRegistry MeasuredRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(MeasuredDoc) }); @@ -483,6 +499,17 @@ namespace GameCult.Caching.Tests path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, countedPayload, "counted.cc"); using (var cache = Open(path, MeasuredRegistry)) AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "measured through its constructor", nameof(Counted), 2, 1); + + // A resolver that hands out MessagePack's own [Key] formatter exempts nothing. + var relayedPayload = Emit((ref MessagePackWriter w) => + { + w.WriteArrayHeader(8); w.Write("r"); + for (var i = 0; i < 6; i++) w.WriteNil(); + w.WriteArrayHeader(2); w.Write("a"); w.Write(Extra); + }); + path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, relayedPayload, "relayed.cc"); + using (var cache = Open(path, MeasuredRegistry)) + AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "relayed by a consumer resolver", nameof(Relayed), 2, 1); } // ---- a flatten writes its base chain's decoded view: refused when a record on the chain is not owned ---- diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index 94597ed2..235e2aef 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -59,8 +59,9 @@ An object's declared slots are the slots its type's own formatter writes for one the highest key plus one); a reader derives them from the layout it writes, never from a list of public members. Only a type the runtime writes by its own positional `[Key]` layout is checked. Exempt, as in the C# reference (`cultlib-gaps:ruling:leaf-acceptance-follows-reference`): a type the runtime formats some other way (a -user codec or formatter, a hand-written serializer, a resolver ahead of the runtime's own, a collection type -written as its items), and a type no instance of which the runtime can write for measuring +user codec or formatter, a hand-written serializer, a resolver ahead of the runtime's own that supplies a layout +of its own, a collection type written as its items; a resolver that hands out the runtime's own positional +layout exempts nothing), and a type no instance of which the runtime can write for measuring (`cultlib-gaps:ruling:slot-unmeasurable-type`). Neither is ever marked. No vector carries an exempt type: each runtime pins its own exemption with its own fixture. Only an array longer than that marks: a short array diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 765d910a..b327f680 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this package are documented in this file. - A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. - `Flatten` (and `FlattenAsync`) of a variant whose base chain holds a record its store does not own (a record held read-only for extra slots, or undeclared drift that drops slots) is refused with `CultSchemaConflictException` naming that base record, and the store is left as it was. Before, the flatten wrote the base's decoded view into a plain record and the base's unread data was lost for that variant. Writing and removing the variant itself are still accepted: they persist only its delta. - `MintElementIds` refuses a home store's commit while that store holds a record read-only for extra slots whose element ids exist only in memory, until the build that declares those slots rewrites it; commits to earlier stores stand. -- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked, and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked. +- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a formatter of its own from a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked; a resolver that hands out MessagePack's own `[Key]`-layout formatter does not exempt the type, and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked. ### Added -- 2.47.3 From b42168484bf1095cbeb33571c03eb4a1d1941b70 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 02:02:19 +0000 Subject: [PATCH 19/20] CHANGELOG: an unwritable type is not checked either (reads as a separate clause) Co-Authored-By: Claude Sonnet 5.5 --- unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index b327f680..a2a1efae 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this package are documented in this file. - A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. - `Flatten` (and `FlattenAsync`) of a variant whose base chain holds a record its store does not own (a record held read-only for extra slots, or undeclared drift that drops slots) is refused with `CultSchemaConflictException` naming that base record, and the store is left as it was. Before, the flatten wrote the base's decoded view into a plain record and the base's unread data was lost for that variant. Writing and removing the variant itself are still accepted: they persist only its delta. - `MintElementIds` refuses a home store's commit while that store holds a record read-only for extra slots whose element ids exist only in memory, until the build that declares those slots rewrites it; commits to earlier stores stand. -- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a formatter of its own from a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked; a resolver that hands out MessagePack's own `[Key]`-layout formatter does not exempt the type, and neither is a type no instance of which can be written (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked. +- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a formatter of its own from a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked; a resolver that hands out MessagePack's own `[Key]`-layout formatter does not exempt the type; a type no instance of which can be written is not checked either (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked. ### Added -- 2.47.3 From 44c798d583eedf1a3bfe5a746481c3243fffb6f3 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 02:20:07 +0000 Subject: [PATCH 20/20] union-parity: outcomes.msgpack stays main's nine rows; the unit arm is pinned only by its store record The bare extra-slot-unit-arm.msgpack and its outcomes row are deleted: the TS, Python and Rust walkers count outcomes.msgpack exactly, so a shared vector does not grow before their leaf cuts. The unit arm's outcome is the unit-arm row of slot-outcomes.msgpack (record unit-arm of extra-slot-store.cc, Denied2 1/0 at depth 1). The README says so. Co-Authored-By: Claude Opus 5.5 --- .../UnionParityVectorTests.cs | 9 +-------- tests/vectors/union-parity/README.md | 13 +++++++------ .../union-parity/extra-slot-unit-arm.msgpack | 1 - tests/vectors/union-parity/outcomes.msgpack | Bin 563 -> 502 bytes 4 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 tests/vectors/union-parity/extra-slot-unit-arm.msgpack diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 067a6cef..9265b2ef 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -297,9 +297,6 @@ namespace GameCult.Caching.Tests Surface(extra: new Extra(RootArm: true))), new Case("extra-slot-list-element.msgpack", "read-only", "Node", 1, "list-element", Surface(extra: new Extra(ListElement: true))), - // A bare UnitOutcome arm (the fixture's unit-arm union), not a UnionSurface payload: [2, [extra]] where Denied2 declares no slot. - new Case("extra-slot-unit-arm.msgpack", "read-only", "UnitOutcome", 2, "unit-arm", - Raw(ArmOf(2, ExtraSlot))), }; private static byte[] OutcomesFile() => Raw((ref MessagePackWriter w) => @@ -320,7 +317,6 @@ namespace GameCult.Caching.Tests "Node" => typeof(Node).FullName!, "Outcome" => typeof(Outcome).FullName!, "CraftedItem" => typeof(CraftedItem).FullName!, - "UnitOutcome" => typeof(UnitOutcome).FullName!, "UnionSurface" => typeof(UnionSurface).FullName!, _ => throw new ArgumentOutOfRangeException(nameof(logical), logical, null) }; @@ -620,10 +616,7 @@ namespace GameCult.Caching.Tests { Assert.That(outcome, Is.EqualTo("read-only"), file); // The extra slot is read past; that the record then loads read-only is asserted in SlotReadOnlyTests, store by store. - if (union == "UnitOutcome") - Assert.That(MessagePackSerializer.Deserialize(payload, UnitOptions), Is.TypeOf(), $"{file}: the extra slot is read past"); - else - Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); + Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); } } } diff --git a/tests/vectors/union-parity/README.md b/tests/vectors/union-parity/README.md index 235e2aef..fbf9c6c1 100644 --- a/tests/vectors/union-parity/README.md +++ b/tests/vectors/union-parity/README.md @@ -52,8 +52,8 @@ Rulings `cultlib-gaps:ruling:extra-trailing-slots` and `cultlib-gaps:ruling:slot slot past its reader's declared slots, in any object at any depth, is read, held read-only, and every write or removal of it is refused before anything is staged. The first four payloads are `union-surface.msgpack` with the string `"extra-slot"` appended as one more slot of one object; a reader decodes the value of `union-surface.msgpack` -from each. The fifth is a bare arm of union `UnitOutcome` (0 Waiting `[Message]`; 2 Denied2, -2147483648 Floor2 and -2147483647 Ceiling2, each `[]`), the union of the `unit-arm-*` files; a reader decodes Denied2 from it. +from each. A unit arm with an extra slot is pinned only as a store record, `unit-arm` in `extra-slot-store.cc` +(below): `outcomes.msgpack` keeps the nine rows the leaf walkers count. An object's declared slots are the slots its type's own formatter writes for one instance (for these fixtures, the highest key plus one); a reader derives them from the layout it writes, never from a list of public members. @@ -74,17 +74,18 @@ key (Image's slot 2) does not mark. A union adds no depth: an arm's array is cou | `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` | nested | Composition, 3, 2 | | `extra-slot-arm.msgpack` | arm | the Root Box arm | nested | Box, 3, 2 | | `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots | nested | Text, 3, 2 | -| `extra-slot-unit-arm.msgpack` | unit-arm | Denied2 in `[2, ["extra-slot"]]` (union `UnitOutcome`, key 2) | as placed | Denied2, 1, 0 | +| record `unit-arm` of `extra-slot-store.cc` | unit-arm | Denied2 in `[2, ["extra-slot"]]` (union `UnitOutcome`, key 2), slot 1 of the document | 1 | Denied2, 1, 0 | | `depth-skipped-at-limit.msgpack` (bounds file) | depth | the document (slot 7, a value of nested arrays) | 0 | UnionSurface, 8, 7 | -Each payload marks exactly one object. Five rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm, -list-element, unit-arm); the sixth `read-only` payload is in `bounds-outcomes.msgpack`. The payload nesting check +Each payload marks exactly one object. Four rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm, +list-element); the fifth `read-only` payload is in `bounds-outcomes.msgpack`, and the unit arm's outcome is its row +in `slot-outcomes.msgpack`. The payload nesting check runs before the mark: `depth-skipped-past-limit.msgpack` is refused, never marked. ### What a runtime asserts for a `read-only` payload Load it as a store record under the reader's own schema id (the C# reference puts each UnionSurface payload in a -`tests.union_surface` store; the unit arm goes in slot 1 of a document `[Name, Outcome: UnitOutcome]`). Then: +`tests.union_surface` store; the unit arm is already a store record, `unit-arm` of `extra-slot-store.cc`). Then: 1. the value is read past the extra slot; 2. the record reports the mark as in the table: the first type read with extra slots, the slots it carried and the diff --git a/tests/vectors/union-parity/extra-slot-unit-arm.msgpack b/tests/vectors/union-parity/extra-slot-unit-arm.msgpack deleted file mode 100644 index 6c780d1d..00000000 --- a/tests/vectors/union-parity/extra-slot-unit-arm.msgpack +++ /dev/null @@ -1 +0,0 @@ -’‘ªextra-slot \ No newline at end of file diff --git a/tests/vectors/union-parity/outcomes.msgpack b/tests/vectors/union-parity/outcomes.msgpack index 217d86c6f70dbb3d62d65b6d263c98fcc034dff0..bbf1a2a0755e5ab3a8e43b245b48d937d2dc797d 100644 GIT binary patch delta 10 RcmdnY@{O5s=0?V^i~ts<1M>g? delta 72 zcmeyyyqSe@)<(v!jJ8vEr&g2{CF&OE