From b091d21f81afb95d4d0626fb48b8f78255aa0a39 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 21:52:36 +0200 Subject: [PATCH 01/47] CultCache: list elements are written by their [Key] layout; registration and the codec refuse otherwise Registration: a concrete element type of an object list that is not keyed, and [MessagePackFormatter] on any type or member on the id path or on the id member, are refused. The id path is a fixed point over the walk's edges. The codec refuses, once per document type and on every encode and decode path, a resolver ahead of StandardResolver that formats a type on that path. UnionArmGuardTests moves StubArm out of the list union. Co-Authored-By: Claude Sonnet 5.5 --- .../CultCacheMessagePack.cs | 4 +- .../CultDocumentMessagePackSerialization.cs | 59 ++++++++-- .../Contracts/cultcache-store-composition.md | 4 + src/GameCult.Caching/CultElementIds.cs | 100 +++++++++++++++-- tests/GameCult.Caching.Tests/AssemblyInfo.cs | 1 + .../ElementIdEngineTests.cs | 59 ++++++++++ .../UnionArmGuardTests.cs | 102 ++++++++++++++++-- 7 files changed, 299 insertions(+), 30 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs index 4fa55076..8d0c7c96 100644 --- a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs +++ b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs @@ -23,10 +23,10 @@ namespace GameCult.Caching.MessagePack // options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type // deserializes as a registered document, a member type as itself. public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec( - (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsForDocument(document)), (type, document, bytes) => type == document ? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry) - : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsForDocument(document)), CultDocumentMessagePackSerialization.OverlaySlots); // The inspection model over that codec. diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index a10a0d62..cba6fdea 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -38,11 +38,23 @@ public static class CultDocumentMessagePackSerialization private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; - public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty()); + private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty()); - private static readonly ConcurrentDictionary AssemblyOptions = new(); + public static readonly MessagePackSerializerOptions Options = Plain.Options; - public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) + // Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of + // StandardResolver, the ones that can claim a type before its [Key] layout does. + private static readonly ConcurrentDictionary AssemblyOptions = new(); + + // Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers + // are fixed for the process, so this is a cache of a pure answer. + private static readonly ConcurrentDictionary DocumentProblems = new(); + + private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!; + + public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options; + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly) { if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly)); return AssemblyOptions.GetOrAdd(documentAssembly, static assembly => @@ -50,19 +62,46 @@ public static class CultDocumentMessagePackSerialization var resolvers = assembly.GetCustomAttributes() .Select(attribute => CreateResolver(attribute.ResolverType)) .ToArray(); - return resolvers.Length == 0 ? Options : Compose(resolvers); + return resolvers.Length == 0 ? Plain : Compose(resolvers); }); } - private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers) + // The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key] + // layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them + // without the id: refused here, once per document type, on every encode and decode path. + internal static MessagePackSerializerOptions OptionsForDocument(Type documentType) + { + if (documentType == null) throw new ArgumentNullException(nameof(documentType)); + var (options, ahead) = ComposedFor(documentType.Assembly); + var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead)); + return problem == null ? options : throw new InvalidOperationException(problem); + } + + private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead) + { + foreach (var type in CultElementIds.Inspect(documentType).IdPath) + { + var generic = GetFormatterMethod.MakeGenericMethod(type); + foreach (var resolver in ahead) + { + if (generic.Invoke(resolver, null) != null) + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout or keep it out of the document's lists."; + } + } + + return null; + } + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers) { // The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer // resolver still encodes an arm while the key check runs. - var inner = CompositeResolver.Create( - consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).Append(StandardResolver.Instance).ToArray()); - return MessagePackSerializerOptions.Standard + var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray(); + var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray()); + var options = MessagePackSerializerOptions.Standard .WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner)) .WithSecurity(CultMessagePackSecurity.Instance); + return (options, ahead); } private static IFormatterResolver CreateResolver(Type resolverType) @@ -84,7 +123,7 @@ public static class CultDocumentMessagePackSerialization { if (registry == null) throw new ArgumentNullException(nameof(registry)); if (value != null) registry.GetRequired(type); - var payload = MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly)); + var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type)); CultPayloadDepth.Check(payload, type, PayloadDirection.Write); return payload; } @@ -99,7 +138,7 @@ public static class CultDocumentMessagePackSerialization if (registry == null) throw new ArgumentNullException(nameof(registry)); registry.GetRequired(type); CultPayloadDepth.Check(payload, type, PayloadDirection.Read); - return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly)) + return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type)) ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index b03627ca..32a92c43 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -294,6 +294,10 @@ exclusive open of the same lock file exclude each other is not established. for `X` that `Doc` needs is declared in `Doc`'s assembly. - An assembly that declares nothing, and the store envelope, use the base options. +- A resolver may not claim a type a document's element lists pass through: the list's + element types, the types and members on the way to them. Those are written by their + `[Key]` layout so each element's id reaches disk, and the codec refuses the document + on first encode or decode, naming the resolver and the type. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 8e10198b..87223289 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -95,34 +95,52 @@ namespace GameCult.Caching internal sealed class Survey { public List Problems { get; } = new(); + + // Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's + // element types and every concrete type under them. A serialization codec must write all of these by their [Key] + // layout, so it checks them against the formatters it composed. + public List IdPath { get; } = new(); + } + + // What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type + // can sit in a cycle, so the path is a fixed point over these edges and not a stack. + private sealed class Graph + { + public List<(Type Owner, MemberInfo? Member, Type Child)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete)> Lists { get; } = new(); } // Every reason this document type cannot be registered under the id rule, one per offending concrete element type. internal static Survey Inspect(Type documentType) { var survey = new Survey(); - VisitObject(documentType, documentType, new HashSet(), survey); + var graph = new Graph(); + VisitObject(documentType, documentType, new HashSet(), survey, graph); + Settle(documentType, survey, graph); return survey; } - private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey) + private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey, Graph graph) { if (!seen.Add(type)) return; foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, seen, survey); + VisitMemberType(documentType, type, member, memberType, seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) - VisitObject(documentType, union.SubType, seen, survey); + { + graph.Edges.Add((type, null, union.SubType)); + VisitObject(documentType, union.SubType, seen, survey, graph); + } } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, Survey survey) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; if (IsLeaf(memberType)) return; if (DictionaryValueType(memberType) is { } valueType) { - VisitMemberType(documentType, owner, member, valueType, seen, survey); + VisitMemberType(documentType, owner, member, valueType, seen, survey, graph); return; } @@ -130,22 +148,80 @@ namespace GameCult.Caching { if (IsObjectType(element)) { - foreach (var concrete in ConcreteTypes(element)) + var concretes = ConcreteTypes(element).ToList(); + graph.Lists.Add((owner, member, element, concretes)); + foreach (var concrete in concretes) { - var problem = IdProblem(concrete); + var problem = IsObjectType(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); } - VisitObject(documentType, element, seen, survey); + VisitObject(documentType, element, seen, survey, graph); } else - VisitMemberType(documentType, owner, member, element, seen, survey); + VisitMemberType(documentType, owner, member, element, seen, survey, graph); return; } if (IsObjectType(memberType)) - VisitObject(documentType, memberType, seen, survey); + { + graph.Edges.Add((owner, member, memberType)); + VisitObject(documentType, memberType, seen, survey, graph); + } + } + + private const string UnkeyedElementProblem = + "is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists."; + + // The id path: every type that reaches an object list, the list's element types and the concrete types under them. + // [MessagePackFormatter] on any of them, or on a member the walk descends through, would hand the bytes to a formatter + // that does not write the [Key] layout the id lives in. + private static void Settle(Type documentType, Survey survey, Graph graph) + { + var reaching = new HashSet(graph.Lists.Select(list => list.Owner)); + var members = new List(graph.Lists.Select(list => list.Member)); + for (var grew = true; grew;) + { + grew = false; + foreach (var (owner, member, child) in graph.Edges) + { + if (!reaching.Contains(child)) + continue; + if (member != null && !members.Contains(member)) + members.Add(member); + grew |= reaching.Add(owner); + } + } + + var path = new List(); + void Add(Type type) + { + if (!path.Contains(type)) + path.Add(type); + } + + foreach (var type in reaching) + Add(type); + foreach (var list in graph.Lists) + { + Add(list.Element); + foreach (var concrete in list.Concrete) + Add(concrete); + } + + foreach (var type in path) + { + survey.IdPath.Add(type); + if (type.IsDefined(typeof(MessagePackFormatterAttribute), true)) + survey.Problems.Add($"Cult document {documentType.Name}: {type.Name} is on the path to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute or keep this type out of the document's lists."); + } + + foreach (var member in members) + { + if (member.IsDefined(typeof(MessagePackFormatterAttribute), true)) + survey.Problems.Add($"Cult document {documentType.Name}: {member.DeclaringType!.Name}.{member.Name} leads to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute."); + } } // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype @@ -177,6 +253,8 @@ namespace GameCult.Caching if (ids.Length > 1) return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity."; var id = ids[0]; + if (id.IsDefined(typeof(MessagePackFormatterAttribute), true)) + return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key]."; if (TypeOf(id) != typeof(string)) return $"declares [CultElementId] on {id.Name}, which is not a string."; if (KeyOf(id) is not { } idKey) diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 0dd8b125..03e1780f 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -6,6 +6,7 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(PairResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] namespace GameCult.Caching.Tests diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 7ae2ee20..bd9ddacd 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -290,12 +290,71 @@ namespace GameCult.Caching.Tests Assert.That(Assert.Throws(() => Assign(twice, "r", deterministic: false))!.Message, Does.Contain("'5'")); } + // A list element is written by its [Key] layout or its id never reaches disk. A union arm without [MessagePackObject] + // is written by some other formatter; a list of a type that is not keyed at all is a value list, replaced whole. + [Test] + public void ANonKeyedArmOfAListUnionIsRefusedAtRegistration() + { + var problem = Problems(typeof(HUnkeyedArmList)).Single(); + Assert.That(problem, Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own").And.Contain("HUnkeyedArmList")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty, "the same union as a field has no element id to lose"); + Assert.That(Problems(typeof(HValueList)), Is.Empty, "a list of a non-keyed type is a value list"); + } + + [Test] + public void AFormatterAttributeOnTheIdPathIsRefused() + { + Assert.That(Problems(typeof(HFormattedElement)).Single(), Does.Contain("FormattedElement").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedList)).Single(), Does.Contain("Items").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedOwner)).Single(), Does.Contain("FormattedOwner").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedMember)).Single(), Does.Contain("Inner").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field"); + } + + // The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and + // a type reached twice, are both on the way to it. + [Test] + public void TheIdPathHoldsEveryTypeThatReachesAnObjectList() + { + var path = CultElementIds.Inspect(typeof(HDiamond)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded) })); + Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded) })); + Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty); + } + // ---- shapes ---- [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } [MessagePackObject] public class ENest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } [MessagePackObject] public class EBareNest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } + public sealed class NoFormatter { } + public class UnkeyedArm : ArmBase { [Key(0)] public string Id { get; set; } = ""; } + [Union(0, typeof(KeyedArm)), Union(1, typeof(UnkeyedArm))] public abstract class ArmBase { } + [MessagePackObject] public class KeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HUnkeyedArmList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HUnkeyedArmField { [Key(0)] public ArmBase? Root { get; set; } } + public class PlainValue { public string X { get; set; } = ""; } + [MessagePackObject] public class HValueList { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedElement { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedElement { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedList { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public List Items { get; set; } = new(); } + [MessagePackObject] public class FormattedIdElement { [Key(0)] [CultElementId] [MessagePackFormatter(typeof(NoFormatter))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedOwner { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedOwner { [Key(0)] public FormattedOwner Owner { get; set; } = new(); } + [MessagePackObject] public class HFormattedMember { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public HHasIds Inner { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedLeaf { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class HFormattedField { [Key(0)] public FormattedLeaf Leaf { get; set; } = new(); [Key(1)] public List Tags { get; set; } = new(); } + + [MessagePackObject] public class HLeft { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HRight { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HCycle { [Key(0)] public HCycleBack Back { get; set; } = new(); } + [MessagePackObject] public class HCycleBack { [Key(0)] public HCycle? Up { get; set; } [Key(1)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HDiamond { [Key(0)] public HLeft Left { get; set; } = new(); [Key(1)] public HRight Right { get; set; } = new(); } + [MessagePackObject] public class HList { [Key(0)] public List Items { get; set; } = new(); } [MessagePackObject] public class HArray { [Key(0)] public EBare[] Items { get; set; } = Array.Empty(); } [MessagePackObject] public class HDictionary { [Key(0)] public Dictionary Map { get; set; } = new(); } diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index a727ba75..21fe44cf 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -20,7 +20,7 @@ namespace GameCult.Caching.Tests { private const string Canary = "CANARY-9d1e47"; - private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc) }); + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc), typeof(StubDoc) }); private delegate void Emit(ref MessagePackWriter writer); @@ -246,15 +246,89 @@ namespace GameCult.Caching.Tests [Test] public void ConsumerResolverStillEncodesArms() { - var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new GuardDoc { Id = "d1", Root = new StubArm() }, typeof(GuardDoc), Registry); - Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Doc((ref MessagePackWriter w) => + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new StubDoc { Id = "d1", Root = new StubArm() }, typeof(StubDoc), Registry); + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Raw((ref MessagePackWriter w) => { + w.WriteArrayHeader(2); w.Write("d1"); w.WriteArrayHeader(2); w.Write(9); w.WriteArrayHeader(1); w.Write("via-resolver"); })))); - var back = (GuardDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(GuardDoc), bytes, Registry); + var back = (StubDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(StubDoc), bytes, Registry); Assert.That(back.Root, Is.TypeOf()); } + // A list element is written by its [Key] layout or its id never reaches disk: a consumer resolver that claims the arm of + // a list's union is refused on every path that encodes or decodes the document, naming the arm and the resolver, and + // writes nothing. + [Test] + public void ListElementsClaimedByAResolverAreRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ClaimDoc) }); + var doc = new ClaimDoc { Id = "c1", Items = { new ClaimArm { N = 4 } } }; + void Refused(TestDelegate attempt) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain("ClaimArm").And.Contain(nameof(ClaimArmResolver)).And.Contain(nameof(ClaimDoc))); + } + + Refused(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ClaimDoc), registry)); + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("c1"); w.WriteArrayHeader(0); }); + Refused(() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ClaimDoc), held, registry)); + Refused(() => CultCacheMessagePack.CreateCodec(registry).Serialize(doc.Items, typeof(List), typeof(ClaimDoc))); + + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "claim.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + Assert.That(Assert.Catch(() => cache.Commit(batch => batch.Upsert(typeof(ClaimDoc), doc, new CultRecordKey("c1"))))!.ToString(), + Does.Contain(nameof(ClaimArmResolver))); + Assert.That(File.Exists(path), Is.False, "nothing reached disk"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + [Union(0, typeof(ClaimArm))] + public abstract class ClaimNode { } + + [MessagePackObject] + public sealed class ClaimArm : ClaimNode + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.claim_doc", "tests.claim_doc.v1")] + [MessagePackObject] + public sealed class ClaimDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + public sealed class ClaimArmResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ClaimArmResolver(); + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(ClaimArm) ? (IMessagePackFormatter)(object)new ClaimFormatter() : null; + + private sealed class ClaimFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, ClaimArm value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(1); + writer.Write(value.N); + } + + public ClaimArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.ReadArrayHeader(); + return new ClaimArm { N = reader.ReadInt32() }; + } + } + } + // The guard wraps the composite, so the composite's own order is still the contract: a consumer resolver outranks the // document resolver (CultRecordRef) and MessagePack's own object formatter, here for a type it also knows. [Test] @@ -568,7 +642,7 @@ namespace GameCult.Caching.Tests } // An arm is [key, armSlots]; the keys here start at 3, skip 0-2, 4, 6 and 7, and never reuse a retired key. - [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(StubArm))] + [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(Nine))] public abstract class GuardNode { } [MessagePackObject] @@ -592,11 +666,25 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class Unit8 : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Nine : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + // A subtype the union does not name. [MessagePackObject] public sealed class Rogue : GuardNode { [Key(0)] public string Note = string.Empty; } - // Written and read by a consumer resolver, not by MessagePack. - public sealed class StubArm : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + // Written and read by a consumer resolver, not by MessagePack. It can only sit outside a list: a list element is written + // by its [Key] layout, or its id never reaches disk (ListElementsClaimedByAResolverAreRefused). + [Union(9, typeof(StubArm))] + public abstract class StubNode { } + + public sealed class StubArm : StubNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + + [CultDocument("tests.stub_doc", "tests.stub_doc.v1")] + [MessagePackObject] + public sealed class StubDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public StubNode? Root; + } public sealed class StubArmResolver : IFormatterResolver { From e5736e49f3a52a006388f1a303fb3cc69988ed7b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 21:52:36 +0200 Subject: [PATCH 02/47] CultCache: a write that copies a record it read holding an element id marks the store The single-file store records, per pull, which plain records decoded holding a non-empty element id and at what version. A write whose copied record is still at that version marks the header (v3). The record is positive-only and version-gated, so nothing a load records can unmark a store. The directory store passes false: only C# writes it. Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 1 + .../Contracts/cultcache-persistence-format.md | 13 +++- src/GameCult.Caching/CultCache.cs | 21 +++++- .../PythonWrittenStoreTests.cs | 65 +++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index eeec3349..76706417 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -363,6 +363,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentManifest.FormatVersion, holdsIds.Values.Select(holds => (bool?)holds).ToArray(), copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)), + copiedHoldIds: false, // only C# writes a directory store, and it marks by content directoryStore: true, holdsVariants: false); WriteManifest(targetCatalog, catalogBytes, currentIndex.Values diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index e1d5d0c0..009fa696 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -423,8 +423,17 @@ elements retargets nothing. the record's next write, or `CultCache.MintElementIds()`. - A store that holds an id declares the `v3` header (above), so a reader older than ids refuses it instead of skipping the id slot and rewriting the elements without ids. - TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they - rewrite a store; they do not decide when a document carries ids. + A C# write also marks the store when it carries forward a record it read holding an + id, as long as the file still holds that record at the version it read. TypeScript, + Python and Rust read `v3` and keep a `v3` header they found when they rewrite a + store; they carry ids and do not decide when a document carries them. +- An object list's elements are written by their `[Key]` layout, because the id is a + keyed member of the element. A list whose element type is not keyed + (`[MessagePackObject]` or `[Union]`) is a value list and is replaced whole. For an + object list, C# refuses at registration a concrete element type that is not keyed and + `[MessagePackFormatter]` on any type or member on the way to the list or on the id + member, and refuses at the first encode or decode of the document a formatter + resolver that claims one of those types. ## Canonical Semantic Schema Hash diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 7f164e4c..4fca998b 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3545,10 +3545,12 @@ namespace GameCult.Caching // The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an // element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An - // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. - protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool directoryStore, bool holdsVariants) => + // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record this store read + // holding an id, at the version the file still holds, marks the store too: another runtime may have written ids under + // an unmarked header. Nothing a load records can unmark a store. + protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) => HeaderFor( - holdsIds: written.Any(value => value == true), + holdsIds: copiedHoldIds || written.Any(value => value == true), durableHeader, wholeStore: !copiesAny && written.All(value => value != null), directoryStore, @@ -3944,6 +3946,10 @@ namespace GameCult.Caching // A write into a store holding a variant lands only while the durable versions still equal these. private readonly Dictionary _lastRead = new(StringComparer.Ordinal); + // The version of each plain record a pull decoded holding an element id. Only a positive fact about those bytes: a + // write marks the header for a copied record still at that version, and nothing here ever unmarks a store. + private readonly Dictionary _readHoldingIds = new(StringComparer.Ordinal); + /// /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its /// file held, one it writes is or of the object. @@ -4032,6 +4038,7 @@ namespace GameCult.Caching if (durable == null) { _lastRead.Clear(); + _readHoldingIds.Clear(); SetLastSchemaMigrationReports(Array.Empty()); SetForeignRecords(Array.Empty()); return; @@ -4042,6 +4049,7 @@ namespace GameCult.Caching var persisted = new Dictionary(StringComparer.Ordinal); var foreign = new List(); var versions = new Dictionary(StringComparer.Ordinal); + var holdingIds = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { var version = versions[record.Key] = durable.Version(record.Key); @@ -4054,6 +4062,8 @@ namespace GameCult.Caching var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; + if (stored.Variant == null && CultElementIds.Holds(stored.Document)) + holdingIds[stored.Key.Value] = version; } var loaded = persisted.Values @@ -4071,6 +4081,9 @@ namespace GameCult.Caching _lastRead.Clear(); foreach (var (key, version) in versions) _lastRead[key] = version; + _readHoldingIds.Clear(); + foreach (var (key, version) in holdingIds) + _readHoldingIds[key] = version; SetLastSchemaMigrationReports(reports); SetForeignRecords(foreign); } @@ -4250,6 +4263,8 @@ namespace GameCult.Caching durable?.Snapshot.FormatVersion, upserts.Select(staged => staged.HoldsIds).ToArray(), copiesAny: copied.Length > 0, + copiedHoldIds: copied.Any(record => _readHoldingIds.TryGetValue(record.Key, out var read) && + SameStoredIdentity(read, durable!.Version(record.Key))), directoryStore: false, holdsVariants: records.Any(record => record.Variant != null)); diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index cc76c760..e20fe926 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -67,6 +67,71 @@ namespace GameCult.Caching.Tests } } + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledAsync(string path) + { + var cache = new CultCache(); + var store = new SingleFileMessagePackBackingStore(path); + cache.AddBackingStore(store); + await cache.PullAllBackingStoresAsync(); + return (cache, store); + } + + // cultcache-py writes element ids under an unmarked v1 header. A C# write that carries such a record forward marks the + // store, because a reader from before ids would otherwise rewrite the elements without them. + [Test] + public async Task ACSharpWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "cultcache-py leaves the header unmarked"); + var (cache, store) = await PulledAsync(path); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // What a load saw is a fact about those bytes at that version. Another writer replaced the record since, so the copy this + // write makes is not the record the load saw, and the header is not judged by it. + [Test] + public async Task ACopiedRecordAnotherWriterChangedIsNotJudgedByTheOldRead() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + var (reader, readerStore) = await PulledAsync(path); + + var (writer, writerStore) = await PulledAsync(path); + var py = writer.AllEntries.OfType().Single(record => record.Id == "py"); + py.Items.Clear(); + await writer.AddAsync(py); + writerStore.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the replacement holds no id"); + + await reader.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + readerStore.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + finally + { + Directory.Delete(scratch, true); + } + } + [TestCase(3)] [TestCase(5)] [TestCase(7)] From 9f20d1be2ab560d85c26a25ae7164487b4af518a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 21:54:49 +0200 Subject: [PATCH 03/47] PythonWrittenStoreTests: the replaced-record test swaps the record's bytes in the file The py record is undeclared by LeafRecord (read-only), so a second cache cannot rewrite it. Co-Authored-By: Claude Sonnet 5.5 --- .../GameCult.Caching.Tests/PythonWrittenStoreTests.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index e20fe926..9ee8a8bc 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -115,11 +115,12 @@ namespace GameCult.Caching.Tests var path = Store("leaf-record.cc", scratch); var (reader, readerStore) = await PulledAsync(path); - var (writer, writerStore) = await PulledAsync(path); - var py = writer.AllEntries.OfType().Single(record => record.Id == "py"); - py.Items.Clear(); - await writer.AddAsync(py); - writerStore.PushAll(); + // Another writer replaces the record with one that holds no id. + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var replaced = snapshot.Records.Single(record => record.Key == "py"); + replaced.Payload = CultDocumentMessagePackSerialization.SerializeUntyped( + new NumericLeafVectorTests.LeafRecord { Id = "py" }, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the replacement holds no id"); await reader.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); From 5ebdc01b326ce4009b0d212073e478046c8a3637 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 22:33:39 +0200 Subject: [PATCH 04/47] Element id tests: the id path holds union bases and inherited formatters; the options entries refuse null Stryker survivors on the id-path walk: dropping the union-subtype edge, dropping the element base from the path. Both now have a test that fails without them. Co-Authored-By: Claude Sonnet 5.5 --- .../ElementIdEngineTests.cs | 15 +++++++++++++++ .../GameCult.Caching.Tests/UnionArmGuardTests.cs | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index bd9ddacd..dd7be0b5 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -310,6 +310,7 @@ namespace GameCult.Caching.Tests Assert.That(Problems(typeof(HFormattedOwner)).Single(), Does.Contain("FormattedOwner").And.Contain("[MessagePackFormatter]")); Assert.That(Problems(typeof(HFormattedMember)).Single(), Does.Contain("Inner").And.Contain("[MessagePackFormatter]")); Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field"); + Assert.That(Problems(typeof(HFormattedDerived)).Single(), Does.Contain("FormattedDerived").And.Contain("[MessagePackFormatter]"), "the attribute is inherited"); } // The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and @@ -321,6 +322,11 @@ namespace GameCult.Caching.Tests Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded) })); Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded) })); Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty); + + // A union base is on the path both as the type a list holds (its formatter writes the arm) and as a field type whose + // arm holds the list. + Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm) })); + Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded) })); } // ---- shapes ---- @@ -349,6 +355,15 @@ namespace GameCult.Caching.Tests [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedLeaf { [Key(0)] public int N { get; set; } } [MessagePackObject] public class HFormattedField { [Key(0)] public FormattedLeaf Leaf { get; set; } = new(); [Key(1)] public List Tags { get; set; } = new(); } + [MessagePackObject] public class FormattedDerived : FormattedElement { } + [MessagePackObject] public class HFormattedDerived { [Key(0)] public List Items { get; set; } = new(); } + [Union(0, typeof(IdArm))] public abstract class IdBase { } + [MessagePackObject] public class IdArm : IdBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Items { get; set; } = new(); } + [Union(0, typeof(PlainArm))] public abstract class PlainBase { } + [MessagePackObject] public class PlainArm : PlainBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HPlainBaseList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HIdBaseOwner { [Key(0)] public IdBase? Root { get; set; } } + [MessagePackObject] public class HLeft { [Key(0)] public HCycle Cycle { get; set; } = new(); } [MessagePackObject] public class HRight { [Key(0)] public HCycle Cycle { get; set; } = new(); } [MessagePackObject] public class HCycle { [Key(0)] public HCycleBack Back { get; set; } = new(); } diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 21fe44cf..eea8c4dc 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -290,6 +290,13 @@ namespace GameCult.Caching.Tests } } + [Test] + public void OptionsRefuseANullDocumentTypeOrAssembly() + { + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsFor(null!)); + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(null!)); + } + [Union(0, typeof(ClaimArm))] public abstract class ClaimNode { } From fb065f0de90e48195056985425dc72fc865be9ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 23:35:31 +0200 Subject: [PATCH 05/47] CultCache: a write decides from the copied records' bytes whether an unmarked store holds an element id; the per-instance read map is deleted _readHoldingIds and the pull's Holds walk are removed in the same commit that adds SingleFileBackingStore.CopiedHoldIds: under the store's lock, while the header on disk is unmarked, every plain copied record the writer's registry claims and whose type reaches an object list is decoded, and the first holding an id marks the store. Positive-only, never refuses a write. CultMesh's typed write attaches the registry its payload was encoded with; the raw write attaches none. Contract text states the rule and its residual. Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 11 ++-- src/GameCult.Caching/CultCache.cs | 66 ++++++++++++++----- src/GameCult.Caching/CultElementIds.cs | 5 ++ .../CultMeshSingleFileDocuments.cs | 17 +++-- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 009fa696..4048eaf8 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -336,8 +336,11 @@ an element id, variants or not). A single-file store declares `v3` when any reco holds carries an element id: a non-empty `[CultElementId]` value in a record's document, or in a variant's override values. A store that cannot read a variant's override values (no codec attached) counts the variant as holding one. An element id is an ordinary keyed -string in the payload, so no reader can tell from a record's bytes whether it holds one; -a writer knows it only for the records it encodes. A write that copies any record forward +string in the payload, so only a reader with the record's type can tell; a C# write +decides for the records it encodes and, while the header it found is unmarked, for every +plain record it copies forward that its registry can read, decoding the bytes the file +holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload, +decide nothing. A write that copies any record forward therefore keeps a `v3` header it found. Only a write that encodes every record the store will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds an id. A directory store's manifest says @@ -423,8 +426,8 @@ elements retargets nothing. the record's next write, or `CultCache.MintElementIds()`. - A store that holds an id declares the `v3` header (above), so a reader older than ids refuses it instead of skipping the id slot and rewriting the elements without ids. - A C# write also marks the store when it carries forward a record it read holding an - id, as long as the file still holds that record at the version it read. TypeScript, + A C# write also marks the store when a record it copies forward holds an id, judged + from the file's bytes at the write (above). TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they rewrite a store; they carry ids and do not decide when a document carries them. - An object list's elements are written by their `[Key]` layout, because the id is a diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 4fca998b..45f1343f 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3491,6 +3491,9 @@ namespace GameCult.Caching protected CultDocumentRegistry Registry => _registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry."); + // Null for a store no registry was attached to (CultMesh's raw single-file write). + private protected CultDocumentRegistry? AttachedRegistry => _registry; + protected ConcurrentDictionary Entries { get; } = new(StringComparer.Ordinal); @@ -3545,9 +3548,9 @@ namespace GameCult.Caching // The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an // element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An - // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record this store read - // holding an id, at the version the file still holds, marks the store too: another runtime may have written ids under - // an unmarked header. Nothing a load records can unmark a store. + // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes + // under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may + // have written ids under an unmarked header. Nothing a load or a decode records can unmark a store. protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) => HeaderFor( holdsIds: copiedHoldIds || written.Any(value => value == true), @@ -3946,10 +3949,6 @@ namespace GameCult.Caching // A write into a store holding a variant lands only while the durable versions still equal these. private readonly Dictionary _lastRead = new(StringComparer.Ordinal); - // The version of each plain record a pull decoded holding an element id. Only a positive fact about those bytes: a - // write marks the header for a copied record still at that version, and nothing here ever unmarks a store. - private readonly Dictionary _readHoldingIds = new(StringComparer.Ordinal); - /// /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its /// file held, one it writes is or of the object. @@ -3974,6 +3973,9 @@ namespace GameCult.Caching // Test seam: how many record encodings this store has hashed, to prove what a write hashes. internal int RecordsHashed; + // Test seam: how many copied records a write decoded to ask whether they hold an element id. + internal int CopiedRecordsDecoded; + private string VersionOfBytes(byte[] bytes) { RecordsHashed++; @@ -4038,7 +4040,6 @@ namespace GameCult.Caching if (durable == null) { _lastRead.Clear(); - _readHoldingIds.Clear(); SetLastSchemaMigrationReports(Array.Empty()); SetForeignRecords(Array.Empty()); return; @@ -4049,7 +4050,6 @@ namespace GameCult.Caching var persisted = new Dictionary(StringComparer.Ordinal); var foreign = new List(); var versions = new Dictionary(StringComparer.Ordinal); - var holdingIds = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { var version = versions[record.Key] = durable.Version(record.Key); @@ -4062,8 +4062,6 @@ namespace GameCult.Caching var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; - if (stored.Variant == null && CultElementIds.Holds(stored.Document)) - holdingIds[stored.Key.Value] = version; } var loaded = persisted.Values @@ -4081,9 +4079,6 @@ namespace GameCult.Caching _lastRead.Clear(); foreach (var (key, version) in versions) _lastRead[key] = version; - _readHoldingIds.Clear(); - foreach (var (key, version) in holdingIds) - _readHoldingIds[key] = version; SetLastSchemaMigrationReports(reports); SetForeignRecords(foreign); } @@ -4219,7 +4214,8 @@ namespace GameCult.Caching // write that then has nothing to write but keys it checked writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions - // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is + // only: a write hashes other records' bytes, never decodes them. That holds for the variant gate; the one decode of a copied + // record is the element-id check (CopiedHoldIds), which runs only on an unmarked store and can only mark it. Otherwise the whole write is refused and what it refused is // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of // the bytes written. @@ -4263,8 +4259,7 @@ namespace GameCult.Caching durable?.Snapshot.FormatVersion, upserts.Select(staged => staged.HoldsIds).ToArray(), copiesAny: copied.Length > 0, - copiedHoldIds: copied.Any(record => _readHoldingIds.TryGetValue(record.Key, out var read) && - SameStoredIdentity(read, durable!.Version(record.Key))), + copiedHoldIds: CopiedHoldIds(durable, copied, upserts), directoryStore: false, holdsVariants: records.Any(record => record.Variant != null)); @@ -4278,6 +4273,43 @@ namespace GameCult.Caching return versions; } + // Whether a record this write copies forward holds an element id, decided from the bytes the file holds under the store's + // lock, so every C# writer with a type for the record decides the same from the same bytes. Positive-only: it runs only + // while the header on disk is unmarked and can only mark the store, so a narrower type that decodes no ids changes nothing. + // It never refuses a write: a record that does not resolve or decode is copied as its bytes and decides nothing. Gates, in + // order, so the decode runs only where it can change the answer. + private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection upserts) + { + if (durable == null || copied.Length == 0 || + string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) || + upserts.Any(staged => staged.HoldsIds == true) || + AttachedRegistry is not { } registry) + return false; + var catalog = durable.Snapshot.SchemaCatalog; + foreach (var record in copied) + { + if (record.Variant != null) + continue; + try + { + if (registry.Claimant(record.SchemaId, catalog, out _) == null) + continue; + var type = registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog).Descriptor.DocumentType; + if (!CultElementIds.CanHoldIds(type)) + continue; + CopiedRecordsDecoded++; + if (CultElementIds.Holds(DeserializePayload(type, record.Payload))) + return true; + } + catch (Exception e) when (e is not OperationCanceledException) + { + // Copied as its bytes: no type, or no decode, is no fact. + } + } + + return false; + } + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { var records = durable?.Snapshot.Records ?? Array.Empty(); diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 87223289..000bed40 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -110,6 +110,11 @@ namespace GameCult.Caching public List<(Type Owner, MemberInfo Member, Type Element, List Concrete)> Lists { get; } = new(); } + private static readonly ConcurrentDictionary IdCapable = new(); + + // A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask. + internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0); + // Every reason this document type cannot be registered under the id rule, one per offending concrete element type. internal static Survey Inspect(Type documentType) { diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 495b2273..17ac6f59 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -108,7 +108,8 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - CultElementIds.Holds(document)); + CultElementIds.Holds(document), + resolvedRegistry); } catch { @@ -146,7 +147,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null); } /// @@ -229,7 +230,8 @@ namespace GameCult.Mesh CultSchemaCatalogEntry catalogEntry, string? storedAt, byte[] payload, - bool? holdsIds) + bool? holdsIds, + CultDocumentRegistry? registry) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -239,8 +241,13 @@ namespace GameCult.Mesh payload ??= Array.Empty(); // The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under // the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null), - // so a file already marked stays marked. A store this runtime cannot read refuses the write instead of being overwritten. - StoreAt(path).ApplyWriteSet( + // so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write + // decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime + // cannot read refuses the write instead of being overwritten. + var store = StoreAt(path); + if (registry != null) + store.AttachRegistry(registry); + store.ApplyWriteSet( new[] { new CultStagedRecord( From 5237b24ab472217cad7fe780c5fb36054ec5509c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 23:35:31 +0200 Subject: [PATCH 06/47] Tests: the copy check marks over ids a writer never read, decodes only what could mark, never refuses; CultMesh typed and raw writes (Soul P8, P8b, P13) Co-Authored-By: Claude Sonnet 5.5 --- .../GameCult.Caching.Tests/ElementIdTests.cs | 67 +++++++++++ .../PythonWrittenStoreTests.cs | 107 ++++++++++++++++++ .../CultMeshSingleFileElementIdTests.cs | 86 ++++++++++++++ 3 files changed, 260 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index c5716975..8dded774 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -1020,6 +1020,73 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1")); } + // Commits one plain item through a cache over `path` and reports how many copied records that write decoded to ask whether + // they hold an element id, and the header it left. + private (int Decoded, string Header) CommitItemCountingDecodes(string path, params Type[] types) + { + var registry = CultDocumentRegistry.ForTypes(types); + var store = new SingleFileMessagePackBackingStore(path); + using var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(store); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + store.CopiedRecordsDecoded = 0; + cache.Commit(batch => batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, new CultRecordKey("p"))); + return (store.CopiedRecordsDecoded, HeaderOf(path)); + } + + // The copy check decodes a record only where the answer can change: each gate, removed, makes one of these counts wrong. + [Test] + public void TheCopyCheckDecodesOnlyWhatCouldMarkTheStore() + { + // A v3 header is already marked: nothing to decide. + var marked = PathOf("gate-marked.cc"); + WritePreIdStore(marked, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(marked)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(marked, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(CommitItemCountingDecodes(marked, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v3")), "a marked store decodes nothing"); + + // Records whose type reaches no object list can hold no id. + var noLists = PathOf("gate-no-lists.cc"); + using (var seed = OpenWith(noLists, false, typeof(PreCut2FixtureItem))) + { + foreach (var key in new[] { "x", "y", "z" }) + UpsertPlain(seed, key); + } + + Assert.That(CommitItemCountingDecodes(noLists, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a type with no object list is not decoded"); + + // A written record that holds an id already marks the store. + var stagedPath = PathOf("gate-staged.cc"); + WritePreIdStore(stagedPath, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var stagedStore = new SingleFileMessagePackBackingStore(stagedPath); + using (var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry))) + { + cache.AddBackingStore(stagedStore); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + stagedStore.CopiedRecordsDecoded = 0; + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh"))); + } + + Assert.That(stagedStore.CopiedRecordsDecoded, Is.EqualTo(0), "a staged record holding an id marks without asking"); + Assert.That(HeaderOf(stagedPath), Is.EqualTo("cultcache.store.v3")); + + // Id-capable records without ids: each is decoded once, and the store stays unmarked. + var capable = PathOf("gate-capable.cc"); + WritePreIdStore(capable, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(capable, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((3, "cultcache.store.v1"))); + + // A record the writer's registry does not claim is foreign: no type, no decode. + var foreign = PathOf("gate-foreign.cc"); + WritePreIdStore(foreign, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(foreign, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a foreign record is not decoded"); + + // A variant is copied as its bytes: only a C# write of one holding ids marks the store, and that write marked it already. + var variant = PathOf("gate-variant.cc"); + WriteV2StoreWithAnIdLessVariant(variant); + Assert.That(CommitItemCountingDecodes(variant, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((1, "cultcache.store.v2")), "the plain base is decoded, the variant is not"); + } + // Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta // it read, so the file holds none and no write of a plain record beside it may mark the file. private static void WriteV2StoreWithAnIdLessVariant(string path) diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index 9ee8a8bc..d7c2acce 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -133,6 +133,113 @@ namespace GameCult.Caching.Tests } } + // A v1 store holding only this record c0, opened by a cache that has pulled it; then another runtime appends the python + // record (and its payload rewritten by `change`) to the file raw, header still v1, the way a writer that keeps the v1 it + // found does. Returns the open cache and its store. + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledThenAppended( + string path, string pySource, Action? change = null) + { + using (var seed = new CultCache()) + { + var seedStore = new SingleFileMessagePackBackingStore(path); + seed.AddBackingStore(seedStore); + await seed.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c0" }); + seedStore.PushAll(); + } + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var opened = await PulledAsync(path); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource)).Records.Single(record => record.Key == "py"); + change?.Invoke(py); + snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found"); + return opened; + } + + // The cache never read this record: it was appended after the pull. A per-instance memory of what the pull saw cannot + // know about it; the write decides from the bytes the file holds under its lock. + [Test] + public async Task ARecordAForeignWriterAddsAfterThePullMarksTheStoreWhenCopied() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "after-pull.cc"); + var (cache, store) = await PulledThenAppended(path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc"))); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // Ids a load minted in memory are not on disk: the check judges the file's bytes, never the document the cache holds. + [Test] + public async Task ARecordWhoseIdsExistOnlyInMemoryLeavesTheStoreUnmarked() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "in-memory.cc"); + var source = Store("leaf-record.cc", scratch); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(source)); + var idless = new NumericLeafVectorTests.LeafRecord { Id = "py" }; + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "one" }); + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "two" }); + snapshot.Records.Single().Payload = CultDocumentMessagePackSerialization.SerializeUntyped(idless, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single().Payload; + + var (cache, store) = await PulledAsync(path); + Assert.That(cache.AllEntries.OfType().Single().Items.Select(item => item.Id), Has.All.Length.EqualTo(12), "the load minted ids in memory"); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py").Payload; + Assert.That(after, Is.EqualTo(before), "the copied record is the bytes the file held"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // The write-set rule copies records without interpreting them, so the check must never refuse a write: a record this + // cache claims but cannot decode is copied as its bytes and decides nothing. + [Test] + public async Task ACopiedRecordThatDoesNotDecodeIsCopiedAndDecidesNothing() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "undecodable.cc"); + var garbage = new byte[] { 0xc1, 0xc1, 0xc1 }; + var (cache, store) = await PulledThenAppended( + path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc")), record => record.Payload = garbage); + + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + Assert.DoesNotThrow(() => store.PushAll()); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py"); + Assert.That(py.Payload, Is.EqualTo(garbage)); + } + finally + { + Directory.Delete(scratch, true); + } + } + [TestCase(3)] [TestCase(5)] [TestCase(7)] diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 0ccf3f17..65a31b2c 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using FluentAssertions; @@ -171,4 +172,89 @@ public sealed class CultMeshSingleFileElementIdTests }); cache.Get(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first); } + + // cultcache-py's record (tests/vectors/numeric-leaves/py-store/leaf-record.cc): v1 header, 12-hex element ids on its list. This + // assembly reads it through the same schema id with the two slots it needs; the member layout is narrower than the writer's. + [CultDocument("cultcache.leaf-record", "cultcache.leaf_record.v1")] + [MessagePackObject] + public sealed class MeshLeaf + { + [Key(0)] [CultName] public string Id { get; set; } = ""; + [Key(12)] public List Items { get; set; } = new(); + } + + [MessagePackObject] + public sealed class MeshLeafItem + { + [Key(0)] public string N { get; set; } = ""; + [Key(1)] public float W { get; set; } + [Key(2)] [CultElementId] public string Id { get; set; } = ""; + } + + private static readonly string[] PythonIds = { "000000000021", "000000000022" }; + + private string PythonStore() + { + string? vectors = null; + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null && vectors == null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "numeric-leaves", "py-store", "leaf-record.cc"); + if (File.Exists(candidate)) vectors = candidate; + } + + vectors.Should().NotBeNull("the shared vectors sit above the test directory"); + var path = Path_("leaf-record.cc"); + File.Copy(vectors!, path); + HeaderOf(path).Should().Be("cultcache.store.v1", "cultcache-py leaves the header unmarked"); + return path; + } + + private static CultDocumentRegistry Registry(params Type[] types) => CultDocumentRegistry.ForTypes(types); + + // A CultMesh write is a writer with a type: it decides for the copied record from the bytes the file holds, whatever any + // earlier read saw (a fresh store per call has read nothing). + [Test] + public void ACultMeshWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore() + { + var path = PythonStore(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshLeaf), typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.ReadSingleFileDocument(path, new CultRecordKey("py")).Items.Select(item => item.Id).Should().Equal(PythonIds); + } + + // Soul s1 P8, committed: the writer's registry has only the type it writes, so the python record is foreign to it and it + // decides nothing for it (variants:question:non-csharp-writers-mark-ids-2). + [Test] + public void ACultMeshWriteWhoseRegistryLacksTheRecordsTypeDecidesNothingForIt() + { + var path = PythonStore(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read. + [Test] + public void ARawCultMeshWriteBesideAPythonRecordHoldingIdsLeavesTheHeaderAlone() + { + var path = PythonStore(); + + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 }); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // The check only marks: a typed write over a store already marked does not look at, and cannot drop, the mark. + [Test] + public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked() + { + var path = Path_("plain.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + CultMesh.WriteSingleFileDocument(path, new CultRecordKey("second"), Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } } From a1c887cde7c0402e3aa5c79163b29b5f9ab200f6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 23:37:10 +0200 Subject: [PATCH 07/47] Tests: the appended python record brings its catalog entry; the Mesh test declares the python document member for member Co-Authored-By: Claude Sonnet 5.5 --- .../PythonWrittenStoreTests.cs | 4 ++- .../CultMeshSingleFileElementIdTests.cs | 36 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index d7c2acce..39c78ded 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -150,9 +150,11 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); var opened = await PulledAsync(path); var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); - var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource)).Records.Single(record => record.Key == "py"); + var pySnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource)); + var py = pySnapshot.Records.Single(record => record.Key == "py"); change?.Invoke(py); snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray(); + snapshot.SchemaCatalog = pySnapshot.SchemaCatalog; File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found"); return opened; diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 65a31b2c..7d2f31c1 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -173,24 +173,44 @@ public sealed class CultMeshSingleFileElementIdTests cache.Get(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first); } - // cultcache-py's record (tests/vectors/numeric-leaves/py-store/leaf-record.cc): v1 header, 12-hex element ids on its list. This - // assembly reads it through the same schema id with the two slots it needs; the member layout is narrower than the writer's. + // cultcache-py's record (tests/vectors/numeric-leaves/py-store/leaf-record.cc): v1 header, 12-hex element ids on its list. + // This assembly declares the same document, member for member, so its registry claims and reads the python record. [CultDocument("cultcache.leaf-record", "cultcache.leaf_record.v1")] [MessagePackObject] public sealed class MeshLeaf { - [Key(0)] [CultName] public string Id { get; set; } = ""; - [Key(12)] public List Items { get; set; } = new(); + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public bool Flag; + [Key(2)] public byte[] Blob = Array.Empty(); + [Key(3)] public int I; + [Key(4)] public long L; + [Key(5)] public float F; + [Key(6)] public double D; + [Key(7)] public int? MaybeI; + [Key(8)] public List Tags = new(); + [Key(9)] public string[] Names = Array.Empty(); + [Key(10)] public object? Any; + [Key(11)] public MeshLeafInner? Nested; + [Key(12)] public List Items = new(); + [Key(13)] public IMeshLeafShape? Shape; + [Key(14)] public float[] Floats = Array.Empty(); + [Key(15)] public float? MaybeF; } [MessagePackObject] - public sealed class MeshLeafItem + public sealed class MeshLeafInner { - [Key(0)] public string N { get; set; } = ""; - [Key(1)] public float W { get; set; } - [Key(2)] [CultElementId] public string Id { get; set; } = ""; + [Key(0)] public string N = string.Empty; + [Key(1)] public float W; + [Key(2)] [CultElementId] public string Id = string.Empty; } + [Union(0, typeof(MeshLeafCircle))] + public interface IMeshLeafShape { } + + [MessagePackObject] + public sealed class MeshLeafCircle : IMeshLeafShape { [Key(0)] public float R; } + private static readonly string[] PythonIds = { "000000000021", "000000000022" }; private string PythonStore() From bd3477450e6e52bfa57fde47fe8624e34d9f8ff4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 23:39:50 +0200 Subject: [PATCH 08/47] Tests: the Mesh test's unmarked file holds ids of a type defined in the Mesh assembly (a persisted layout is matched by CLR type name); the appended python record brings its catalog entry beside the cache's Co-Authored-By: Claude Sonnet 5.5 --- .../PythonWrittenStoreTests.cs | 2 +- .../CultMeshSingleFileElementIdTests.cs | 88 +++++++++---------- 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index 39c78ded..f83ae98d 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -154,7 +154,7 @@ namespace GameCult.Caching.Tests var py = pySnapshot.Records.Single(record => record.Key == "py"); change?.Invoke(py); snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray(); - snapshot.SchemaCatalog = pySnapshot.SchemaCatalog; + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Concat(pySnapshot.SchemaCatalog).GroupBy(entry => entry.SchemaId).Select(group => group.First()).ToArray(); File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found"); return opened; diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 7d2f31c1..c4f86d1e 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -173,45 +173,41 @@ public sealed class CultMeshSingleFileElementIdTests cache.Get(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first); } - // cultcache-py's record (tests/vectors/numeric-leaves/py-store/leaf-record.cc): v1 header, 12-hex element ids on its list. - // This assembly declares the same document, member for member, so its registry claims and reads the python record. - [CultDocument("cultcache.leaf-record", "cultcache.leaf_record.v1")] + // A list document defined in this assembly: the record another writer leaves holding ids under an unmarked header. + [CultDocument("mesh.test.id_list", "mesh.test.id_list.v1")] [MessagePackObject] - public sealed class MeshLeaf + public sealed class MeshIdList { - [Key(0)] [CultName] public string Id = string.Empty; - [Key(1)] public bool Flag; - [Key(2)] public byte[] Blob = Array.Empty(); - [Key(3)] public int I; - [Key(4)] public long L; - [Key(5)] public float F; - [Key(6)] public double D; - [Key(7)] public int? MaybeI; - [Key(8)] public List Tags = new(); - [Key(9)] public string[] Names = Array.Empty(); - [Key(10)] public object? Any; - [Key(11)] public MeshLeafInner? Nested; - [Key(12)] public List Items = new(); - [Key(13)] public IMeshLeafShape? Shape; - [Key(14)] public float[] Floats = Array.Empty(); - [Key(15)] public float? MaybeF; + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Items { get; set; } = new(); } [MessagePackObject] - public sealed class MeshLeafInner + public sealed class MeshIdItem { - [Key(0)] public string N = string.Empty; - [Key(1)] public float W; - [Key(2)] [CultElementId] public string Id = string.Empty; + [Key(0)] public string N { get; set; } = ""; + [Key(1)] [CultElementId] public string Id { get; set; } = ""; } - [Union(0, typeof(MeshLeafCircle))] - public interface IMeshLeafShape { } + private static readonly CultRecordKey IdsKey = new("ids"); + private const string HexB = "bbbbbbbbbbbb"; - [MessagePackObject] - public sealed class MeshLeafCircle : IMeshLeafShape { [Key(0)] public float R; } - - private static readonly string[] PythonIds = { "000000000021", "000000000022" }; + // A v1 file whose record holds 12-hex element ids: the document is written with the type (v3), then the header is put back + // to the v1 a writer that does not decide would have left. + private string UnmarkedStoreHoldingIds() + { + var path = Path_("unmarked.cc"); + CultMesh.WriteSingleFileDocument( + path, + IdsKey, + new MeshIdList { Name = "ids", Items = { new MeshIdItem { Id = HexA }, new MeshIdItem { Id = HexB } } }, + registry: Registry(typeof(MeshIdList))); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + HeaderOf(path).Should().Be("cultcache.store.v1"); + return path; + } private string PythonStore() { @@ -234,14 +230,25 @@ public sealed class CultMeshSingleFileElementIdTests // A CultMesh write is a writer with a type: it decides for the copied record from the bytes the file holds, whatever any // earlier read saw (a fresh store per call has read nothing). [Test] - public void ACultMeshWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore() + public void ACultMeshWriteThatCopiesARecordHoldingIdsMarksTheStore() { - var path = PythonStore(); + var path = UnmarkedStoreHoldingIds(); - CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshLeaf), typeof(CultMeshBodyPublicationDocument))); + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument))); HeaderOf(path).Should().Be("cultcache.store.v3"); - CultMesh.ReadSingleFileDocument(path, new CultRecordKey("py")).Items.Select(item => item.Id).Should().Equal(PythonIds); + CultMesh.ReadSingleFileDocument(path, IdsKey, Registry(typeof(MeshIdList))).Items.Select(item => item.Id).Should().Equal(HexA, HexB); + } + + // A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read. + [Test] + public void ARawCultMeshWriteBesideARecordHoldingIdsLeavesTheHeaderAlone() + { + var path = UnmarkedStoreHoldingIds(); + + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 }); + + HeaderOf(path).Should().Be("cultcache.store.v1"); } // Soul s1 P8, committed: the writer's registry has only the type it writes, so the python record is foreign to it and it @@ -256,18 +263,7 @@ public sealed class CultMeshSingleFileElementIdTests HeaderOf(path).Should().Be("cultcache.store.v1"); } - // A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read. - [Test] - public void ARawCultMeshWriteBesideAPythonRecordHoldingIdsLeavesTheHeaderAlone() - { - var path = PythonStore(); - - CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 }); - - HeaderOf(path).Should().Be("cultcache.store.v1"); - } - - // The check only marks: a typed write over a store already marked does not look at, and cannot drop, the mark. + // Typed writes beside records that hold no id leave the store unmarked: the check marks only on an id it found. [Test] public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked() { From b6977e276e362212bef90e1e639992271d1d0103 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 23:48:21 +0200 Subject: [PATCH 09/47] ElementIdTests: the staged-holds-an-id gate is pinned with two written records, only one holding an id Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/ElementIdTests.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 8dded774..70201d19 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -1056,16 +1056,21 @@ namespace GameCult.Caching.Tests Assert.That(CommitItemCountingDecodes(noLists, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a type with no object list is not decoded"); - // A written record that holds an id already marks the store. + // A written record that holds an id already marks the store, whichever of the written records it is. var stagedPath = PathOf("gate-staged.cc"); WritePreIdStore(stagedPath, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); var stagedStore = new SingleFileMessagePackBackingStore(stagedPath); - using (var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry))) + var stagedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck), typeof(PreCut2FixtureItem) }); + using (var cache = new CultCache(stagedRegistry, CultCacheMessagePack.CreateCodec(stagedRegistry))) { cache.AddBackingStore(stagedStore); cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); stagedStore.CopiedRecordsDecoded = 0; - cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh"))); + cache.Commit(batch => + { + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh")); + }); } Assert.That(stagedStore.CopiedRecordsDecoded, Is.EqualTo(0), "a staged record holding an id marks without asking"); From e118f0995f621f139861e8c8115f4f382c999fa6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:23:17 +0200 Subject: [PATCH 10/47] CultMesh element-id tests: an unresolvable copied record is copied and decides nothing (Soul s2 P5) Co-Authored-By: Claude Sonnet 5.5 --- .../CultMeshSingleFileElementIdTests.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index c4f86d1e..2863925e 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -263,6 +263,29 @@ public sealed class CultMeshSingleFileElementIdTests HeaderOf(path).Should().Be("cultcache.store.v1"); } + // The copy check never refuses a write: a record the registry claims by schema name but cannot resolve (the catalog entry + // names another type for a slot) is copied as its bytes and decides nothing. Soul s2 P5, incompatible-catalog-slot12. + [Test] + public void AnUnresolvableCopiedRecordIsCopiedAndDecidesNothing() + { + var path = UnmarkedStoreHoldingIds(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + const string changedId = "mesh.test.id_list.v0"; + var entry = snapshot.SchemaCatalog.Single(candidate => candidate.SchemaName == "mesh.test.id_list"); + entry.SchemaId = changedId; + entry.Members.Single(member => member.Slot == 1).TypeName = "System.Collections.Generic.List`1[[Other.Inner]]"; + snapshot.Records.Single(record => record.Key == IdsKey.Value).SchemaId = changedId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + + Assert.DoesNotThrow(() => CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument)))); + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + after.SchemaId.Should().Be(changedId); + after.Payload.Should().Equal(before.Payload); + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + // Typed writes beside records that hold no id leave the store unmarked: the check marks only on an id it found. [Test] public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked() From fa90c6cc1ebf121c79aa6e5c499b845e1e94ac4b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:23:25 +0200 Subject: [PATCH 11/47] CopiedHoldIds: the copied.Length == 0 early-out is deleted; an empty copy runs the loop zero times Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 45f1343f..9d491704 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -4280,7 +4280,7 @@ namespace GameCult.Caching // order, so the decode runs only where it can change the answer. private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection upserts) { - if (durable == null || copied.Length == 0 || + if (durable == null || string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) || upserts.Any(staged => staged.HoldsIds == true) || AttachedRegistry is not { } registry) From 328e9b37e1b310f2b3ae0d283c35795254e2f6eb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:23:25 +0200 Subject: [PATCH 12/47] HeaderForWrite comment: only a reader with the record's type can tell from its bytes whether it holds an element id Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 9d491704..c90fdfcc 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3545,8 +3545,9 @@ namespace GameCult.Caching return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1; } - // The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an - // element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any + // The header a write leaves, for either store kind: HeaderFor decides. An element id is an ordinary keyed string, so only a + // reader with the record's type can tell from its bytes whether it holds one; the store header is how a reader without the types + // learns it. A write that copies any // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes // under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may From 73b4b47451cffdfb410cdb14dcbcdcfa4562dc85 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:26:52 +0200 Subject: [PATCH 13/47] CultElementIds: collection types join the id path, member attributes read through overrides, each unkeyed element type refused once Graph edges and lists carry the collection types between member and child; Settle adds them to the path so [MessagePackFormatter] on a collection class and a resolver claim on List are judged. Declares replaces the four MemberInfo.IsDefined member reads, which ignore inherit for properties. Survey.Judged dedupes the unkeyed-element refusal. Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 2 +- src/GameCult.Caching/CultElementIds.cs | 41 ++++--- .../ElementIdEngineTests.cs | 108 +++++++++++++++++- 3 files changed, 135 insertions(+), 16 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 4048eaf8..f1335e8f 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -434,7 +434,7 @@ elements retargets nothing. keyed member of the element. A list whose element type is not keyed (`[MessagePackObject]` or `[Union]`) is a value list and is replaced whole. For an object list, C# refuses at registration a concrete element type that is not keyed and - `[MessagePackFormatter]` on any type or member on the way to the list or on the id + `[MessagePackFormatter]` on any type, collection type or member on the way to the list or on the id member, and refuses at the first encode or decode of the document a formatter resolver that claims one of those types. diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 000bed40..de457f8a 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -100,14 +100,17 @@ namespace GameCult.Caching // element types and every concrete type under them. A serialization codec must write all of these by their [Key] // layout, so it checks them against the formatters it composed. public List IdPath { get; } = new(); + + // The concrete element types already judged in this document, so each is refused at most once however many lists hold it. + internal HashSet Judged { get; } = new(); } // What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type // can sit in a cycle, so the path is a fixed point over these edges and not a stack. private sealed class Graph { - public List<(Type Owner, MemberInfo? Member, Type Child)> Edges { get; } = new(); - public List<(Type Owner, MemberInfo Member, Type Element, List Concrete)> Lists { get; } = new(); + public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through)> Lists { get; } = new(); } private static readonly ConcurrentDictionary IdCapable = new(); @@ -130,22 +133,22 @@ namespace GameCult.Caching if (!seen.Add(type)) return; foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, seen, survey, graph); + VisitMemberType(documentType, type, member, memberType, Array.Empty(), seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) { - graph.Edges.Add((type, null, union.SubType)); + graph.Edges.Add((type, null, union.SubType, Array.Empty())); VisitObject(documentType, union.SubType, seen, survey, graph); } } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, Survey survey, Graph graph) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; if (IsLeaf(memberType)) return; if (DictionaryValueType(memberType) is { } valueType) { - VisitMemberType(documentType, owner, member, valueType, seen, survey, graph); + VisitMemberType(documentType, owner, member, valueType, through.Append(memberType).ToArray(), seen, survey, graph); return; } @@ -154,9 +157,11 @@ namespace GameCult.Caching if (IsObjectType(element)) { var concretes = ConcreteTypes(element).ToList(); - graph.Lists.Add((owner, member, element, concretes)); + graph.Lists.Add((owner, member, element, concretes, through.Append(memberType).ToArray())); foreach (var concrete in concretes) { + if (!survey.Judged.Add(concrete)) + continue; var problem = IsObjectType(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); @@ -165,13 +170,13 @@ namespace GameCult.Caching VisitObject(documentType, element, seen, survey, graph); } else - VisitMemberType(documentType, owner, member, element, seen, survey, graph); + VisitMemberType(documentType, owner, member, element, through.Append(memberType).ToArray(), seen, survey, graph); return; } if (IsObjectType(memberType)) { - graph.Edges.Add((owner, member, memberType)); + graph.Edges.Add((owner, member, memberType, through)); VisitObject(documentType, memberType, seen, survey, graph); } } @@ -186,15 +191,17 @@ namespace GameCult.Caching { var reaching = new HashSet(graph.Lists.Select(list => list.Owner)); var members = new List(graph.Lists.Select(list => list.Member)); + var through = new List(graph.Lists.SelectMany(list => list.Through)); for (var grew = true; grew;) { grew = false; - foreach (var (owner, member, child) in graph.Edges) + foreach (var (owner, member, child, via) in graph.Edges) { if (!reaching.Contains(child)) continue; if (member != null && !members.Contains(member)) members.Add(member); + through.AddRange(via); grew |= reaching.Add(owner); } } @@ -208,6 +215,8 @@ namespace GameCult.Caching foreach (var type in reaching) Add(type); + foreach (var type in through) + Add(type); foreach (var list in graph.Lists) { Add(list.Element); @@ -224,7 +233,7 @@ namespace GameCult.Caching foreach (var member in members) { - if (member.IsDefined(typeof(MessagePackFormatterAttribute), true)) + if (Declares(member, typeof(MessagePackFormatterAttribute))) survey.Problems.Add($"Cult document {documentType.Name}: {member.DeclaringType!.Name}.{member.Name} leads to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute."); } } @@ -252,13 +261,13 @@ namespace GameCult.Caching private static string? IdProblem(Type element) { - var ids = MembersOf(element).Where(member => member.IsDefined(typeof(CultElementIdAttribute), true)).ToArray(); + var ids = MembersOf(element).Where(member => Declares(member, typeof(CultElementIdAttribute))).ToArray(); if (ids.Length == 0) return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity."; if (ids.Length > 1) return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity."; var id = ids[0]; - if (id.IsDefined(typeof(MessagePackFormatterAttribute), true)) + if (Declares(id, typeof(MessagePackFormatterAttribute))) return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key]."; if (TypeOf(id) != typeof(string)) return $"declares [CultElementId] on {id.Name}, which is not a string."; @@ -475,7 +484,7 @@ namespace GameCult.Caching private static Shape Build(Type type) { var shape = new Shape(); - var idMember = MembersOf(type).FirstOrDefault(member => member.IsDefined(typeof(CultElementIdAttribute), true)); + var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute))); if (idMember != null) { shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o); @@ -504,6 +513,10 @@ namespace GameCult.Caching private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member))); + // An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an + // override of a base virtual member would not see its base's attribute. + private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); + private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType; diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index dd7be0b5..551faada 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -313,6 +313,52 @@ namespace GameCult.Caching.Tests Assert.That(Problems(typeof(HFormattedDerived)).Single(), Does.Contain("FormattedDerived").And.Contain("[MessagePackFormatter]"), "the attribute is inherited"); } + // The collection types between a member and its object list are written by some formatter too, so they sit on the id path. + // A value list (of a struct, a primitive or a non-keyed class) never reaches an object, so it does not. + [Test] + public void ACollectionTypeOnTheWayToAnObjectListIsOnTheIdPath() + { + var path = CultElementIds.Inspect(typeof(HCollections)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] + { + typeof(HCollections), typeof(EIded), typeof(HHasIds), + typeof(List), typeof(EIded[]), typeof(Dictionary), typeof(List>) + })); + + var valueLists = CultElementIds.Inspect(typeof(HValueListsBesideObjects)).IdPath; + Assert.That(valueLists, Is.EquivalentTo(new[] { typeof(HValueListsBesideObjects), typeof(List), typeof(EIded) })); + } + + [Test] + public void AFormatterOnACollectionClassOfAnObjectListIsRefused() + { + Assert.That(Problems(typeof(HRoles)).Single(), Does.Contain("Roles").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HFormattedValueCollection)), Is.Empty, "a collection class of a non-keyed element is a value list"); + } + + // MemberInfo.IsDefined ignores inherit for properties: an override sees its base's attribute only through Declares. + [Test] + public void AnInheritedFormatterOnAnOverriddenMemberIsRefused() + { + Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(OwnerOverFormatted)).Single(), Does.Contain("Items").And.Contain("[MessagePackFormatter]")); + + Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id"); + var root = new HIdOverride { Items = { new IdOverride() } }; + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(root.Items[0].Id, Has.Length.EqualTo(12)); + } + + // The union arm has a valid id, so IdProblem passes it and only the rule that an element is written by its [Key] layout + // can refuse it; it is refused once per document however many lists hold it. + [Test] + public void ANonKeyedArmWithAValidIdIsRefusedOncePerType() + { + Assert.That(Problems(typeof(HUnkeyedArmList)).Single(), Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own")); + Assert.That(Problems(typeof(HUnkeyedTwoLists)).Single(), Does.Contain("UnkeyedArm")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty); + } + // The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and // a type reached twice, are both on the way to it. [Test] @@ -336,11 +382,71 @@ namespace GameCult.Caching.Tests [MessagePackObject] public class EBareNest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } public sealed class NoFormatter { } - public class UnkeyedArm : ArmBase { [Key(0)] public string Id { get; set; } = ""; } + public class UnkeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } [Union(0, typeof(KeyedArm)), Union(1, typeof(UnkeyedArm))] public abstract class ArmBase { } [MessagePackObject] public class KeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } [MessagePackObject] public class HUnkeyedArmList { [Key(0)] public List Items { get; set; } = new(); } [MessagePackObject] public class HUnkeyedArmField { [Key(0)] public ArmBase? Root { get; set; } } + [MessagePackObject] + public class HCollections + { + [Key(0)] public List A { get; set; } = new(); + [Key(1)] public EIded[] B { get; set; } = Array.Empty(); + [Key(2)] public Dictionary C { get; set; } = new(); + [Key(3)] public List> D { get; set; } = new(); + } + + [MessagePackObject] + public class HValueListsBesideObjects + { + [Key(0)] public List Objects { get; set; } = new(); + [Key(1)] public List Numbers { get; set; } = new(); + [Key(2)] public List Plain { get; set; } = new(); + [Key(3)] public List Points { get; set; } = new(); + } + + [MessagePackFormatter(typeof(NoFormatter))] public class Roles : List { } + [MessagePackObject] public class HRoles { [Key(0)] public Roles Items { get; set; } = new(); } + [MessagePackFormatter(typeof(NoFormatter))] public class PlainValues : List { } + [MessagePackObject] public class HFormattedValueCollection { [Key(0)] public List Objects { get; set; } = new(); [Key(1)] public PlainValues Plain { get; set; } = new(); } + + [MessagePackObject] + public abstract class FormattedIdBase + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] + public class FormattedIdOverride : FormattedIdBase + { + [Key(0)] [CultElementId] public override string Id { get; set; } = ""; + } + + [MessagePackObject] public class HOverFormattedId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class OwnerBaseFormatted + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual List Items { get; set; } = new(); + } + + [MessagePackObject] + public class OwnerOverFormatted : OwnerBaseFormatted + { + [Key(0)] public override List Items { get; set; } = new(); + } + + [MessagePackObject] + public class IdVirtual + { + [Key(0)] public string X { get; set; } = ""; + [Key(1)] [CultElementId] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] public class IdOverride : IdVirtual { [Key(1)] public override string Id { get; set; } = ""; } + [MessagePackObject] public class HIdOverride { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HUnkeyedTwoLists { [Key(0)] public List A { get; set; } = new(); [Key(1)] public List B { get; set; } = new(); } public class PlainValue { public string X { get; set; } = ""; } [MessagePackObject] public class HValueList { [Key(0)] public List Items { get; set; } = new(); } From b0cc36bc7ec9e8e5e8b38ad2dc51b99afe600aaa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:26:53 +0200 Subject: [PATCH 14/47] ResolverClaim: a resolver returning MessagePack's standard formatter for an id-path type claims nothing Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 11 +- .../Contracts/cultcache-store-composition.md | 10 +- tests/GameCult.Caching.Tests/AssemblyInfo.cs | 1 + .../UnionArmGuardTests.cs | 164 ++++++++++++++++++ 4 files changed, 181 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index cba6fdea..96a4b985 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -82,10 +82,17 @@ public static class CultDocumentMessagePackSerialization foreach (var type in CultElementIds.Inspect(documentType).IdPath) { var generic = GetFormatterMethod.MakeGenericMethod(type); + // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the + // composite stops there, so no later resolver is asked. + var standard = generic.Invoke(StandardResolver.Instance, null); foreach (var resolver in ahead) { - if (generic.Invoke(resolver, null) != null) - return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout or keep it out of the document's lists."; + var formatter = generic.Invoke(resolver, null); + if (formatter == null) + continue; + if (ReferenceEquals(formatter, standard)) + break; + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout or keep it out of the document's lists."; } } diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 32a92c43..545d9a8f 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -295,9 +295,13 @@ exclusive open of the same lock file exclude each other is not established. - An assembly that declares nothing, and the store envelope, use the base options. - A resolver may not claim a type a document's element lists pass through: the list's - element types, the types and members on the way to them. Those are written by their - `[Key]` layout so each element's id reaches disk, and the codec refuses the document - on first encode or decode, naming the resolver and the type. + element types, the types and members on the way to them, and the collection types + the walk passes (`List`, `T[]`, a collection class, a dictionary on the way). + Those are written by their `[Key]` layout so each element's id reaches disk, and the + codec refuses the document on first encode or decode, naming the resolver and the + type. A resolver that returns MessagePack's standard formatter for a type does not + claim it. A value list (`List`, a list of a non-keyed class) is not on the + path, so a resolver may claim it. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 03e1780f..c63262f6 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -8,6 +8,7 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index eea8c4dc..aab52ec4 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -290,6 +290,170 @@ namespace GameCult.Caching.Tests } } + // A resolver's formatter for the list type of an object list drops the elements' ids as surely as one for the element, + // so the collection type is on the id path (Soul s1 P1). A claim on a value list, even beside an object list, is none. + [Test] + public void AResolverClaimingTheListTypeOfAnObjectListIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(LcDoc), typeof(LcValueDoc) }); + var doc = new LcDoc { Id = "l1", Items = { new LcArm { Id = "000000000001", N = 4 } } }; + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("l1"); w.WriteArrayHeader(0); }); + foreach (var attempt in new TestDelegate[] + { + () => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(LcDoc), registry), + () => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcDoc), held, registry) + }) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain("List`1").And.Contain(nameof(LcDoc))); + } + + var values = new LcValueDoc + { + Id = "v1", + Items = { new LcOther { Id = "000000000002" } }, + Values = { new LcValue { N = 3 }, new LcValue { N = 5 } }, + Points = { new Pair { A = 1, B = 2 } } + }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(values, typeof(LcValueDoc), registry); + var back = (LcValueDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcValueDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000002")); + Assert.That(back.Values.Select(value => value.N), Is.EqualTo(new[] { 3, 5 })); + Assert.That(back.Points.Single(), Is.EqualTo(new Pair { A = 1, B = 2 })); + } + + // A resolver that hands out MessagePack's own formatter for a type writes its [Key] layout: it claims nothing, however it + // got there, and the document encodes with the id present (Soul s1 P2). + [Test] + public void AResolverHandingBackTheStandardFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(FbDoc) }); + var doc = new FbDoc { Id = "f1", Items = { new FbArm { Id = "000000000009", N = 7 } } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(FbDoc), registry); + + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(doc, CultDocumentMessagePackSerialization.Options)))); + var back = (FbDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(FbDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000009")); + } + + [MessagePackObject] + public sealed class LcArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [MessagePackObject] + public sealed class LcOther + { + [Key(0)] [CultElementId] public string Id = string.Empty; + } + + public sealed class LcValue + { + public int N; + } + + [CultDocument("tests.lc_doc", "tests.lc_doc.v1")] + [MessagePackObject] + public sealed class LcDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [CultDocument("tests.lc_value_doc", "tests.lc_value_doc.v1")] + [MessagePackObject] + public sealed class LcValueDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + [Key(2)] public List Values = new(); + [Key(3)] public List Points = new(); + } + + [MessagePackObject] + public sealed class FbArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.fb_doc", "tests.fb_doc.v1")] + [MessagePackObject] + public sealed class FbDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + // Claims List (an object list's type: refused), List and List (value lists: accepted, as a math + // resolver claims List), and answers FbArm with MessagePack's own formatter (no claim). + public sealed class ListClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ListClaimResolver(); + + public IMessagePackFormatter? GetFormatter() + { + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new NilFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); + if (typeof(T) == typeof(FbArm)) return StandardResolver.Instance.GetFormatter(); + return null; + } + + private sealed class NilFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new List(); + } + } + + private sealed class ValuesFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) writer.Write(item.N); + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) list.Add(new LcValue { N = reader.ReadInt32() }); + return list; + } + } + + private sealed class PointsFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) { writer.WriteArrayHeader(2); writer.Write(item.A); writer.Write(item.B); } + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) + { + reader.ReadArrayHeader(); + list.Add(new Pair { A = reader.ReadSingle(), B = reader.ReadSingle() }); + } + + return list; + } + } + } + [Test] public void OptionsRefuseANullDocumentTypeOrAssembly() { From 95ddc59680913672107b9b040630b622c69dc9b8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:28:30 +0200 Subject: [PATCH 15/47] CultMesh element-id test: the catalog entry's content hash differs, so the record really is unresolvable Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 2863925e..a8c501de 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -273,6 +273,7 @@ public sealed class CultMeshSingleFileElementIdTests const string changedId = "mesh.test.id_list.v0"; var entry = snapshot.SchemaCatalog.Single(candidate => candidate.SchemaName == "mesh.test.id_list"); entry.SchemaId = changedId; + entry.ContentHash = "persisted-by-another-runtime"; entry.Members.Single(member => member.Slot == 1).TypeName = "System.Collections.Generic.List`1[[Other.Inner]]"; snapshot.Records.Single(record => record.Key == IdsKey.Value).SchemaId = changedId; File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); From d46e000b5c61cabd50455489418dff906f83558d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:31:38 +0200 Subject: [PATCH 16/47] UnionArmGuardTests: the standard-formatter resolver names MessagePack.Resolvers.StandardResolver Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index aab52ec4..722c1663 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -399,7 +399,7 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new NilFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); - if (typeof(T) == typeof(FbArm)) return StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(FbArm)) return MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); return null; } From 56e25d245b96e1da7e74426f7444344b38bf106f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:33:46 +0200 Subject: [PATCH 17/47] UnionArmGuardTests: global:: reaches MessagePack.Resolvers from inside GameCult.Caching.Tests Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 722c1663..7b79235a 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -399,7 +399,7 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new NilFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); - if (typeof(T) == typeof(FbArm)) return MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); return null; } From 9eb43be7f15a43812522f34b37422e052abb6a08 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:34:39 +0200 Subject: [PATCH 18/47] TheIdPathHoldsEveryTypeThatReachesAnObjectList: the expected paths now include the list types the walk passes Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/ElementIdEngineTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 551faada..e91ae8b1 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -365,14 +365,14 @@ namespace GameCult.Caching.Tests public void TheIdPathHoldsEveryTypeThatReachesAnObjectList() { var path = CultElementIds.Inspect(typeof(HDiamond)).IdPath; - Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded) })); - Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded) })); + Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded), typeof(List), typeof(List) })); Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty); // A union base is on the path both as the type a list holds (its formatter writes the arm) and as a field type whose // arm holds the list. - Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm) })); - Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded) })); + Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded), typeof(List) })); } // ---- shapes ---- From c9c96642e86d8b74625faa35dbc10226b72fb9cb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 00:35:57 +0200 Subject: [PATCH 19/47] UnionArmGuardTests: a later resolver's claim on a type the standard formatter already answered is never asked Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/AssemblyInfo.cs | 1 + .../UnionArmGuardTests.cs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index c63262f6..708f0a09 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -9,6 +9,7 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 7b79235a..9fae36b6 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -454,6 +454,26 @@ namespace GameCult.Caching.Tests } } + // Composed after ListClaimResolver: once that resolver has answered FbArm with the standard formatter the composite never + // asks this one, so its claim on the same type must not refuse the document. + public sealed class FbLaterClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new FbLaterClaimResolver(); + + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(FbArm) ? (IMessagePackFormatter)(object)new LaterFormatter() : null; + + private sealed class LaterFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FbArm value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public FbArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new FbArm(); + } + } + } + [Test] public void OptionsRefuseANullDocumentTypeOrAssembly() { From 1e7b3b25da39281d2998f19d4a923537f45508a5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:21:22 +0200 Subject: [PATCH 20/47] Element ids: one Classify of the declared type; registration, Plan and Holds walk declared types IsLeaf, IsObjectType and DictionaryValueType are deleted. Classify(declared) says Leaf, Map (IDictionary<,> or IReadOnlyDictionary<,>), Sequence or Owner; a keyed struct is walked as an owner and is never an element. Plan and Holds take the declared root type and follow declared member types, so a member declared object or a subclass's extra members under a base-typed member are neither minted into nor read. Minted path text is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 15 +- src/GameCult.Caching/CultElementIds.cs | 226 +++++++++++------- .../CultMeshSingleFileDocuments.cs | 6 +- .../ElementIdEngineTests.cs | 136 +++++++++-- 4 files changed, 270 insertions(+), 113 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index c90fdfcc..c037626d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -545,11 +545,14 @@ namespace GameCult.Caching get { if (Variant == null) - return CultElementIds.Holds(_document); + return CultElementIds.Holds(Descriptor.DocumentType, _document); if (_codec == null) return true; - return Variant.Overrides.Any(entry => CultElementIds.Holds(_codec.Deserialize( - Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType, Descriptor.DocumentType, entry.Value))); + return Variant.Overrides.Any(entry => + { + var memberType = Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType; + return CultElementIds.Holds(memberType, _codec.Deserialize(memberType, Descriptor.DocumentType, entry.Value)); + }); } } @@ -2734,7 +2737,7 @@ namespace GameCult.Caching // again, and lands in memory only (variants mint inside Resolve, into their override values). var idPlans = admitted .Where(stored => stored.Variant == null) - .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Document, stored.Key.Value, deterministic: source != null))) + .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Descriptor.DocumentType, stored.Document, stored.Key.Value, deterministic: source != null))) .ToArray(); VariantPlan plan; CultCommitOutcome outcome; @@ -3176,7 +3179,7 @@ namespace GameCult.Caching // The override's value is minted like a plain record's members: ids the store never held are filled here, in the // decoded copy, and the delta this hold lands carries them. var bytes = entry.Value; - var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); + var idPlan = CultElementIds.Plan(member.MemberType, value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); if (idPlan.Count > 0) { idPlan.Apply(); @@ -4299,7 +4302,7 @@ namespace GameCult.Caching if (!CultElementIds.CanHoldIds(type)) continue; CopiedRecordsDecoded++; - if (CultElementIds.Holds(DeserializePayload(type, record.Payload))) + if (CultElementIds.Holds(type, DeserializePayload(type, record.Payload))) return true; } catch (Exception e) when (e is not OperationCanceledException) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index de457f8a..adf8db01 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -85,7 +85,7 @@ namespace GameCult.Caching public Action? SetId; public Func? Derive; public string? DerivedFrom; - public (int Slot, Func Get)[] Walk = Array.Empty<(int, Func)>(); + public (int Slot, Type Type, Func Get)[] Walk = Array.Empty<(int, Type, Func)>(); } private static readonly ConcurrentDictionary Shapes = new(); @@ -144,40 +144,35 @@ namespace GameCult.Caching private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; - if (IsLeaf(memberType)) - return; - if (DictionaryValueType(memberType) is { } valueType) + var (kind, inner) = Classify(memberType); + switch (kind) { - VisitMemberType(documentType, owner, member, valueType, through.Append(memberType).ToArray(), seen, survey, graph); - return; - } - - if (ElementTypeOf(memberType) is { } element) - { - if (IsObjectType(element)) - { - var concretes = ConcreteTypes(element).ToList(); - graph.Lists.Add((owner, member, element, concretes, through.Append(memberType).ToArray())); + case Kind.Leaf: + return; + case Kind.Map: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + case Kind.Sequence when IsElement(inner): + var concretes = ConcreteTypes(inner).ToList(); + graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray())); foreach (var concrete in concretes) { if (!survey.Judged.Add(concrete)) continue; - var problem = IsObjectType(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; + var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); } - VisitObject(documentType, element, seen, survey, graph); - } - else - VisitMemberType(documentType, owner, member, element, through.Append(memberType).ToArray(), seen, survey, graph); - return; - } - - if (IsObjectType(memberType)) - { - graph.Edges.Add((owner, member, memberType, through)); - VisitObject(documentType, memberType, seen, survey, graph); + VisitObject(documentType, inner, seen, survey, graph); + return; + case Kind.Sequence: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + default: + graph.Edges.Add((owner, member, memberType, through)); + VisitObject(documentType, memberType, seen, survey, graph); + return; } } @@ -295,40 +290,37 @@ namespace GameCult.Caching // ---- write: what the stored form holds ---- // True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value - // as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. - internal static bool Holds(object? root) + // as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. The walk + // follows declared types, as registration does; a value's own type picks only a union arm. + internal static bool Holds(Type declared, object? root) { if (root == null) return false; - var type = root.GetType(); - if (IsLeaf(type)) - return false; - if (root is IDictionary dictionary) + var (kind, inner) = Classify(declared); + switch (kind) { - foreach (DictionaryEntry entry in dictionary) - { - if (Holds(entry.Value)) - return true; - } + case Kind.Leaf: + return false; + case Kind.Map: + return MapEntries(declared)(root).Any(entry => Holds(inner, entry.Value)); + case Kind.Sequence: + var elements = IsElement(inner); + if (!elements && Classify(inner).Kind == Kind.Leaf) + return false; + foreach (var item in (IEnumerable)root) + { + if (item == null) + continue; + if (elements && OwnerShape(inner, item).GetId is { } getId && !string.IsNullOrEmpty(getId(item))) + return true; + if (Holds(inner, item)) + return true; + } - return false; + return false; + default: + return OwnerShape(declared, root).Walk.Any(member => Holds(member.Type, member.Get(root))); } - - if (root is IEnumerable list) - { - foreach (var item in list) - { - if (item != null && IsObjectType(item.GetType()) && ShapeOf(item.GetType()).GetId is { } getId && - !string.IsNullOrEmpty(getId(item))) - return true; - if (Holds(item)) - return true; - } - - return false; - } - - return IsObjectType(type) && ShapeOf(type).Walk.Any(member => Holds(member.Get(root))); } // ---- write and load: mint and check ---- @@ -337,41 +329,48 @@ namespace GameCult.Caching // that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty. // rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits // twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load. - internal static IdPlan Plan(object? root, string rootPath, bool deterministic) + internal static IdPlan Plan(Type declared, object? root, string rootPath, bool deterministic) { var plan = new IdPlan(); - Visit(root, rootPath); + Visit(declared, root, rootPath); return plan; - void Visit(object? value, string path) + void Visit(Type type, object? value, string path) { if (value == null) return; - var type = value.GetType(); - if (IsLeaf(type)) - return; - if (value is IDictionary dictionary) + var (kind, inner) = Classify(type); + switch (kind) { - foreach (DictionaryEntry entry in dictionary) - Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); - } - else if (value is IEnumerable list) - Elements(list, path); - else if (IsObjectType(type)) - { - foreach (var (slot, get) in ShapeOf(type).Walk) - Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); + case Kind.Map: + foreach (var entry in MapEntries(type)(value)) + Visit(inner, entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); + break; + case Kind.Sequence when IsElement(inner): + Elements(inner, (IEnumerable)value, path); + break; + case Kind.Sequence: + if (Classify(inner).Kind == Kind.Leaf) + break; + var index = 0; + foreach (var item in (IEnumerable)value) + Visit(inner, item, path + "/" + (index++).ToString(CultureInfo.InvariantCulture)); + break; + case Kind.Owner: + foreach (var (slot, memberType, get) in OwnerShape(type, value).Walk) + Visit(memberType, get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); + break; } } - void Elements(IEnumerable list, string path) + void Elements(Type element, IEnumerable list, string path) { var items = list.Cast().ToArray(); var taken = new HashSet(StringComparer.Ordinal); var objects = new HashSet(ReferenceComparer.Instance); foreach (var item in items) { - if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape) + if (item == null || OwnerShape(element, item) is not { GetId: { } getId } shape) continue; if (!objects.Add(item)) throw new CultElementIdException( @@ -394,7 +393,7 @@ namespace GameCult.Caching if (item == null) continue; var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture); - if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape && + if (OwnerShape(element, item) is { SetId: { } , GetId: { } get } itemShape && string.IsNullOrEmpty(get(item))) { string id; @@ -413,7 +412,7 @@ namespace GameCult.Caching plan.Add(item, itemShape, get(item), id); } - Visit(item, itemPath); + Visit(element, item, itemPath); } } @@ -481,6 +480,14 @@ namespace GameCult.Caching private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build); + // The shape of an owner or element a walk holds under a declared type: a union base writes its arm, so the value's own + // type picks the arm; any other declared type is written by its own [Key] layout, whatever a subclass adds. + private static Shape OwnerShape(Type declared, object value) + { + declared = Nullable.GetUnderlyingType(declared) ?? declared; + return declared.IsDefined(typeof(UnionAttribute), false) ? ShapeOf(value.GetType()) : ShapeOf(declared); + } + private static Shape Build(Type type) { var shape = new Shape(); @@ -498,8 +505,8 @@ namespace GameCult.Caching } shape.Walk = KeyedMembers(type) - .Where(entry => !IsLeaf(Nullable.GetUnderlyingType(entry.Type) ?? entry.Type)) - .Select(entry => (Slot: KeyOf(entry.Member)!.Value, Get: entry.Member is FieldInfo fi + .Where(entry => Classify(entry.Type).Kind != Kind.Leaf) + .Select(entry => (Slot: KeyOf(entry.Member)!.Value, entry.Type, Get: entry.Member is FieldInfo fi ? (Func)(o => fi.GetValue(o)) : o => ((PropertyInfo)entry.Member).GetValue(o))) .ToArray(); @@ -523,18 +530,38 @@ namespace GameCult.Caching // ---- type classification ---- - // A value MessagePack writes whole and that cannot hold an element list. Structs are values: never elements, never walked. - private static bool IsLeaf(Type type) - { - if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type.IsValueType) - return true; - return type.IsArray && IsLeaf(type.GetElementType()!); - } + private enum Kind { Leaf, Map, Sequence, Owner } - // An object a store can hold an element list under, or hold as an element: a class MessagePack serializes by key. - private static bool IsObjectType(Type type) => - (type.IsClass || type.IsInterface) && type != typeof(string) && type != typeof(object) && - (type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true)); + private static readonly ConcurrentDictionary Classes = new(); + + // The one recognition of what MessagePack writes for a declared type: registration, Plan, Holds and the codec's id path + // all go through it. Inner is a map's value type, a sequence's element type, or the type itself for an owner. + // Leaf a value written whole that holds no element list: primitives, enums, strings, object (untyped on disk), + // a struct with no [MessagePackObject] or [Union], and a class with neither. + // Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>. + // Sequence an array or an IEnumerable. + // Owner a class, interface or struct MessagePack serializes by key. A keyed struct is walked as an owner; it is + // never an element (IsElement), since a boxed copy cannot carry a minted id. + private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type => + { + type = Nullable.GetUnderlyingType(type) ?? type; + if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type == typeof(object)) + return (Kind.Leaf, type); + if (type.IsValueType) + return IsKeyed(type) ? (Kind.Owner, type) : (Kind.Leaf, type); + if (MapTypes(type) is { } map) + return (Kind.Map, map.Value); + if (ElementTypeOf(type) is { } element) + return (Kind.Sequence, element); + return IsKeyed(type) ? (Kind.Owner, type) : (Kind.Leaf, type); + }); + + // A type MessagePack serializes by key: [MessagePackObject] or [Union]. + private static bool IsKeyed(Type type) => + type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true); + + // What an object list holds: a keyed class or interface. Never a struct. + private static bool IsElement(Type type) => !type.IsValueType && Classify(type).Kind == Kind.Owner; private static Type? ElementTypeOf(Type type) { @@ -549,15 +576,34 @@ namespace GameCult.Caching return null; } - private static Type? DictionaryValueType(Type type) + // The key and value types of a map MessagePack writes as a map: IDictionary<,> or IReadOnlyDictionary<,>. + private static (Type Key, Type Value)? MapTypes(Type type) { foreach (var candidate in type.GetInterfaces().Append(type)) { - if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>)) - return candidate.GetGenericArguments()[1]; + if (candidate.IsGenericType && + (candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>) || candidate.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>))) + { + var arguments = candidate.GetGenericArguments(); + return (arguments[0], arguments[1]); + } } return null; } + + private static readonly ConcurrentDictionary>> MapReaders = new(); + + // A map's entries read through the generic interface Classify found, so no map shape is recognised twice. + private static Func> MapEntries(Type map) => MapReaders.GetOrAdd(map, static type => + { + var (key, value) = MapTypes(type)!.Value; + return (Func>)Delegate.CreateDelegate( + typeof(Func>), + typeof(CultElementIds).GetMethod(nameof(ReadMap), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(key, value)); + }); + + private static IEnumerable<(object? Key, object? Value)> ReadMap(object map) => + ((IEnumerable>)map).Select(entry => ((object?)entry.Key, (object?)entry.Value)); } } diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 17ac6f59..40480895 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -98,7 +98,7 @@ namespace GameCult.Mesh var descriptor = resolvedRegistry.GetRequired(); // Element ids are decided as a cache write decides them, before anything is written: an unset id is minted, a // refused list refuses the write, and the file is marked when it holds an id. A failed write gives the ids back. - var ids = CultElementIds.Plan(document, key.Value, deterministic: false); + var ids = CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: false); ids.Apply(); try { @@ -108,7 +108,7 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - CultElementIds.Holds(document), + CultElementIds.Holds(typeof(TDocument), document), resolvedRegistry); } catch @@ -132,7 +132,7 @@ namespace GameCult.Mesh var payload = ReadSingleFileDocumentPayload(path, key, descriptor.SchemaId); var document = (TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), payload); // A load refuses what a write refuses, and mints the ids a cache load of the same file would mint. - CultElementIds.Plan(document, key.Value, deterministic: true).Apply(); + CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: true).Apply(); return document; } diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index e91ae8b1..608ef47d 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -16,7 +16,7 @@ namespace GameCult.Caching.Tests private static int Assign(object root, string key, bool deterministic) { - var plan = CultElementIds.Plan(root, key, deterministic); + var plan = CultElementIds.Plan(root.GetType(), root, key, deterministic); plan.Apply(); return plan.Count; } @@ -30,7 +30,8 @@ namespace GameCult.Caching.Tests foreach (var holder in new[] { typeof(HList), typeof(HArray), typeof(HDictionary), typeof(HNested), typeof(HListOfLists), - typeof(HElementNest), typeof(HUnion), typeof(HNullableInner) + typeof(HElementNest), typeof(HUnion), typeof(HNullableInner), typeof(HStruct), typeof(HStructList), typeof(HNullableStruct), + typeof(HReadOnlyMap) }) RefusesBare(holder); Assert.That(Problems(typeof(HFine)), Is.Empty); @@ -72,28 +73,28 @@ namespace GameCult.Caching.Tests public void HoldsFindsASetIdWhereverAListCanSitAndNothingElse() { const string id = "0123456789ab"; - Assert.That(CultElementIds.Holds(null), Is.False); - Assert.That(CultElementIds.Holds(new HAll + Assert.That(CultElementIds.Holds(typeof(HAll), null), Is.False); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded() } } }, Grid = { new List { new EIded() } }, Nest = { new ENest { Sub = { new EIded() } } }, }), Is.False, "unset ids hold nothing"); - Assert.That(CultElementIds.Holds(new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); - Assert.That(CultElementIds.Holds(new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); } [Test] public void APlanCountsAndChangesNothingUntilApplied() { var root = new HHasIds { Items = { new EIded(), new EIded() } }; - var plan = CultElementIds.Plan(root, "r", deterministic: true); + var plan = CultElementIds.Plan(typeof(HHasIds), root, "r", deterministic: true); Assert.That(plan.Count, Is.EqualTo(2)); Assert.That(root.Items.Select(x => x.Id), Has.All.Empty); plan.Apply(); @@ -184,7 +185,7 @@ namespace GameCult.Caching.Tests Assert.That(root.Dict["k"].Items.Select(x => x.Id), Has.All.Empty, "the list before the refused one was not minted into"); var ok = new HHasIds { Items = { new EIded(), new EIded { Id = "dddddddddddd" } } }; - var plan = CultElementIds.Plan(ok, "r", deterministic: false); + var plan = CultElementIds.Plan(typeof(HHasIds), ok, "r", deterministic: false); plan.Apply(); plan.Undo(); Assert.That(ok.Items.Select(x => x.Id), Is.EqualTo(new[] { "", "dddddddddddd" })); @@ -375,6 +376,82 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded), typeof(List) })); } + // A keyed struct is a value MessagePack writes by key, and an object list inside one carries element ids like any other. + // It is walked as an owner and is never an element: a boxed copy cannot carry a minted id. + [Test] + public void AKeyedStructIsWalkedAsAnOwnerAndNeverAnElement() + { + var root = new HStructIded + { + Hold = new SHoldIded { Items = new List { new EIded() } }, + Many = { new SHoldIded { Items = new List { new EIded() } } }, + }; + Assert.That(Problems(typeof(HStructIded)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructIded)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructIded), typeof(SHoldIded), typeof(List), typeof(List), typeof(EIded) })); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.False); + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(root.Hold.Items[0].Id, Is.Not.Empty, "visible through the struct copy"); + Assert.That(root.Many[0].Items[0].Id, Is.Not.Empty, "visible through the struct in a list"); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.True); + var one = new HStructIded { Hold = new SHoldIded { Items = new List { new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Holds(typeof(HStructIded), one), Is.True); + + Assert.That(Problems(typeof(HStructValues)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructValues)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructValues), typeof(List), typeof(EIded) }), "a list of keyed structs is a value list"); + var values = new HStructValues { Structs = { new SValue { N = 1 } }, Objects = { new EIded() } }; + Assert.That(Assign(values, "r", deterministic: false), Is.EqualTo(1)); + } + + // Every dictionary interface MessagePack writes as a map is walked by one path, and the minted path text is the one a + // store loaded before ids already holds, so its deterministic ids do not move. + [Test] + public void EveryMapInterfaceIsWalkedByTheSamePath() + { + var element = new Func(() => new HHasIds { Items = { new EIded() } }); + var viaInterface = new HReadOnlyIded { Map = new Dictionary { ["k"] = element() } }; + var viaClass = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = element() }) }; + var viaDictionary = new HMapIded { Map = { ["k"] = element() } }; + Assert.That(Assign(viaInterface, "r", deterministic: true), Is.EqualTo(1)); + Assert.That(Assign(viaClass, "r", deterministic: true), Is.EqualTo(1), "a class implementing only IReadOnlyDictionary<,>"); + Assert.That(Assign(viaDictionary, "r", deterministic: true), Is.EqualTo(1)); + var id = viaDictionary.Map["k"].Items[0].Id; + Assert.That(viaInterface.Map["k"].Items[0].Id, Is.EqualTo(id)); + Assert.That(viaClass.Map["k"].Items[0].Id, Is.EqualTo(id)); + + // SHA-256 of "||0": slot 0, the key in braces, slot 0, the index. + var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("r|r.0{k}.0/0|0")); + Assert.That(id, Is.EqualTo(Convert.ToHexString(hash, 0, 6).ToLowerInvariant()), "the path text is unchanged"); + + var held = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = new HHasIds { Items = { new EIded { Id = "0123456789ab" } } } }) }; + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), held), Is.True); + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), viaInterface), Is.True); + Assert.That(CultElementIds.Holds(typeof(HMapIded), new HMapIded { Map = { ["k"] = new HHasIds { Items = { new EIded() } } } }), Is.False); + } + + // Registration judges the declared shape, so minting and reading follow it: a member declared object is untyped on disk, + // and what a subclass adds under a base-typed member is not written, so neither is walked. + [Test] + public void TheWalkFollowsTheDeclaredShape() + { + var derived = new HBase { B = new DerC { L = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HBase), derived, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HBase), derived), Is.False); + var opaque = new HObject { O = new HHasIds { Items = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HObject), opaque, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HObject), opaque), Is.False); + foreach (var type in new[] { typeof(HBase), typeof(HObject) }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + // A union base still writes its arm, so the arm is walked. + var arm = new HIdBaseOwner { Root = new IdArm { Items = { new EIded() } } }; + Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1)); + } + // ---- shapes ---- [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } @@ -517,6 +594,37 @@ namespace GameCult.Caching.Tests [Key(2)] public List Nest { get; set; } = new(); } + [MessagePackObject] public struct SHold { [Key(0)] public List Items; } + [MessagePackObject] public struct SHoldIded { [Key(0)] public List Items; } + [MessagePackObject] public struct SValue { [Key(0)] public int N; } + [MessagePackObject] public class HStruct { [Key(0)] public SHold Hold; } + [MessagePackObject] public class HStructList { [Key(0)] public List Holds { get; set; } = new(); } + [MessagePackObject] public class HNullableStruct { [Key(0)] public SHold? Hold; } + [MessagePackObject] public class HStructIded { [Key(0)] public SHoldIded Hold; [Key(1)] public List Many = new(); } + [MessagePackObject] public class HStructValues { [Key(0)] public List Structs = new(); [Key(1)] public List Objects = new(); } + + [MessagePackObject] public class HReadOnlyMap { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HReadOnlyIded { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HMapIded { [Key(0)] public Dictionary Map { get; set; } = new(); } + public sealed class RoMap : IReadOnlyDictionary + { + private readonly Dictionary _inner; + public RoMap(Dictionary inner) => _inner = inner; + public HHasIds this[string key] => _inner[key]; + public IEnumerable Keys => _inner.Keys; + public IEnumerable Values => _inner.Values; + public int Count => _inner.Count; + public bool ContainsKey(string key) => _inner.ContainsKey(key); + public bool TryGetValue(string key, out HHasIds value) => _inner.TryGetValue(key, out value!); + public IEnumerator> GetEnumerator() => _inner.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [MessagePackObject] public class BaseC { [Key(0)] public int X { get; set; } } + [MessagePackObject] public class DerC : BaseC { [Key(1)] public List L { get; set; } = new(); } + [MessagePackObject] public class HBase { [Key(0)] public BaseC B { get; set; } = new(); } + [MessagePackObject] public class HObject { [Key(0)] public object? O { get; set; } } + [MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; } [MessagePackObject] public class HDerivedText { [Key(0)] public List Items { get; set; } = new(); } From 143e4f7db25f3da43e1f5bc6a5f318a1f36a1582 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:22:47 +0200 Subject: [PATCH 21/47] Element ids: a non-public [Key] member on the way to an object list is refused at registration MessagePack writes a non-public [Key] member only under AllowPrivate, and the element-id walk reads public members, so the ids such a list holds could never be checked. Graph entries carry Hidden; a hidden list or a hidden edge whose child reaches adds one problem and nothing to the id path. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultElementIds.cs | 63 ++++++++++++++----- .../ElementIdEngineTests.cs | 27 ++++++++ 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index adf8db01..fcbf8079 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -109,8 +109,10 @@ namespace GameCult.Caching // can sit in a cycle, so the path is a fixed point over these edges and not a stack. private sealed class Graph { - public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new(); - public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through)> Lists { get; } = new(); + // Hidden is true when the member that starts the entry is non-public: MessagePack writes it only under AllowPrivate, + // and the walk reads public members only. + public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through, bool Hidden)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through, bool Hidden)> Lists { get; } = new(); } private static readonly ConcurrentDictionary IdCapable = new(); @@ -133,15 +135,17 @@ namespace GameCult.Caching if (!seen.Add(type)) return; foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, Array.Empty(), seen, survey, graph); + VisitMemberType(documentType, type, member, memberType, Array.Empty(), false, seen, survey, graph); + foreach (var (member, memberType) in NonPublicKeyedMembers(type)) + VisitMemberType(documentType, type, member, memberType, Array.Empty(), true, seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) { - graph.Edges.Add((type, null, union.SubType, Array.Empty())); + graph.Edges.Add((type, null, union.SubType, Array.Empty(), false)); VisitObject(documentType, union.SubType, seen, survey, graph); } } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, bool hidden, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; var (kind, inner) = Classify(memberType); @@ -150,14 +154,15 @@ namespace GameCult.Caching case Kind.Leaf: return; case Kind.Map: - VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), hidden, seen, survey, graph); return; case Kind.Sequence when IsElement(inner): var concretes = ConcreteTypes(inner).ToList(); - graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray())); + graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray(), hidden)); foreach (var concrete in concretes) { - if (!survey.Judged.Add(concrete)) + // A hidden list is refused as a member (Settle), not once per element type. + if (hidden || !survey.Judged.Add(concrete)) continue; var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) @@ -167,10 +172,10 @@ namespace GameCult.Caching VisitObject(documentType, inner, seen, survey, graph); return; case Kind.Sequence: - VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), hidden, seen, survey, graph); return; default: - graph.Edges.Add((owner, member, memberType, through)); + graph.Edges.Add((owner, member, memberType, through, hidden)); VisitObject(documentType, memberType, seen, survey, graph); return; } @@ -184,15 +189,16 @@ namespace GameCult.Caching // that does not write the [Key] layout the id lives in. private static void Settle(Type documentType, Survey survey, Graph graph) { - var reaching = new HashSet(graph.Lists.Select(list => list.Owner)); - var members = new List(graph.Lists.Select(list => list.Member)); - var through = new List(graph.Lists.SelectMany(list => list.Through)); + var lists = graph.Lists.Where(list => !list.Hidden).ToArray(); + var reaching = new HashSet(lists.Select(list => list.Owner)); + var members = new List(lists.Select(list => list.Member)); + var through = new List(lists.SelectMany(list => list.Through)); for (var grew = true; grew;) { grew = false; - foreach (var (owner, member, child, via) in graph.Edges) + foreach (var (owner, member, child, via, hidden) in graph.Edges) { - if (!reaching.Contains(child)) + if (hidden || !reaching.Contains(child)) continue; if (member != null && !members.Contains(member)) members.Add(member); @@ -212,7 +218,7 @@ namespace GameCult.Caching Add(type); foreach (var type in through) Add(type); - foreach (var list in graph.Lists) + foreach (var list in lists) { Add(list.Element); foreach (var concrete in list.Concrete) @@ -231,8 +237,18 @@ namespace GameCult.Caching if (Declares(member, typeof(MessagePackFormatterAttribute))) survey.Problems.Add($"Cult document {documentType.Name}: {member.DeclaringType!.Name}.{member.Name} leads to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute."); } + + // The fixed point has settled which types reach a list through public members. A non-public [Key] member that starts + // a list, or leads to a type that reaches one, is refused: nothing reads or checks the ids it holds. + foreach (var list in graph.Lists.Where(list => list.Hidden)) + survey.Problems.Add(HiddenProblem(documentType, list.Owner, list.Member)); + foreach (var edge in graph.Edges.Where(edge => edge.Hidden && reaching.Contains(edge.Child))) + survey.Problems.Add(HiddenProblem(documentType, edge.Owner, edge.Member!)); } + private static string HiddenProblem(Type documentType, Type owner, MemberInfo member) => + $"Cult document {documentType.Name}: {owner.Name}.{member.Name} is a non-public [Key] member on the way to a list whose elements carry ids; MessagePack writes it only under AllowPrivate and the element-id walk reads public members, so its ids could never be checked. Make it public, or keep lists out of it."; + // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype // under it. Shapes are built from these, so registration judges exactly what minting will meet. private static IEnumerable ConcreteTypes(Type element) @@ -524,6 +540,21 @@ namespace GameCult.Caching // override of a base virtual member would not see its base's attribute. private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); + // [Key] members MessagePack writes only under AllowPrivate: non-public fields and properties, up the base chain. + private static IEnumerable<(MemberInfo Member, Type Type)> NonPublicKeyedMembers(Type type) + { + for (var level = type; level != null && level != typeof(object); level = level.BaseType) + { + const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + foreach (var member in level.GetFields(flags).Cast() + .Concat(level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0))) + { + if (KeyOf(member) != null) + yield return (member, TypeOf(member)); + } + } + } + private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType; diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 608ef47d..4e9bb460 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -452,6 +452,24 @@ namespace GameCult.Caching.Tests Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1)); } + // MessagePack writes a non-public [Key] member only under AllowPrivate, and the element-id walk reads public members, so + // an object list a non-public member leads to could never be checked. It is refused; a non-public member that reaches no + // object list is nobody's business. + [Test] + public void ANonPublicKeyedMemberOnTheWayToAnObjectListIsRefused() + { + foreach (var (type, member) in new[] + { + (typeof(HHiddenList), "Stash"), (typeof(HHiddenOwner), "stash"), (typeof(HOptHidden), "stash"), (typeof(HHiddenFromBase), "stash"), + }) + Assert.That(Problems(type).Single(), Does.Contain(member).And.Contain("non-public"), type.Name); + + Assert.That(Problems(typeof(HHiddenLeaf)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HHiddenLeaf)).IdPath, Is.EquivalentTo(new[] { typeof(HHiddenLeaf), typeof(List), typeof(EIded) }), + "a hidden member adds nothing to the id path"); + Assert.That(CultElementIds.Inspect(typeof(HHiddenList)).IdPath, Is.Empty); + } + // ---- shapes ---- [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } @@ -625,6 +643,15 @@ namespace GameCult.Caching.Tests [MessagePackObject] public class HBase { [Key(0)] public BaseC B { get; set; } = new(); } [MessagePackObject] public class HObject { [Key(0)] public object? O { get; set; } } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class HHiddenList { [Key(0)] private List Stash { get; set; } = new(); } + [MessagePackObject] public class HHiddenOwner { [Key(0)] private HHasIds stash = new(); } + [MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List stash = new(); } + [MessagePackObject] public class HHiddenBase { [Key(0)] private List stash = new(); } + [MessagePackObject] public class HHiddenFromBase : HHiddenBase { [Key(1)] public int N { get; set; } } + [MessagePackObject] public class HHiddenLeaf { [Key(0)] public List Items { get; set; } = new(); [Key(1)] private int secret = 7; } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; } [MessagePackObject] public class HDerivedText { [Key(0)] public List Items { get; set; } = new(); } From 908d9ac29d79e3011a77c7142f8dcf5812561db7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:23:40 +0200 Subject: [PATCH 22/47] Codec: StandardResolverAllowPrivate's formatter for an id-path type is no claim It writes the [Key] layout plus non-public keyed members, and registration now refuses any of those that reaches an object list, so no element id can be dropped. The comparison is made lazily, only after an ahead resolver returned a non-standard formatter, and an exception from it reads as not equal. Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 21 ++++++- .../UnionArmGuardTests.cs | 60 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 96a4b985..ec0326fb 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -83,8 +83,12 @@ public static class CultDocumentMessagePackSerialization { var generic = GetFormatterMethod.MakeGenericMethod(type); // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the - // composite stops there, so no later resolver is asked. + // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout + // plus non-public [Key] members, and registration refuses a non-public member on the way to an object list, so it + // claims nothing either. var standard = generic.Invoke(StandardResolver.Instance, null); + var allowPrivateAsked = false; + object? allowPrivate = null; foreach (var resolver in ahead) { var formatter = generic.Invoke(resolver, null); @@ -92,6 +96,21 @@ public static class CultDocumentMessagePackSerialization continue; if (ReferenceEquals(formatter, standard)) break; + if (!allowPrivateAsked) + { + allowPrivateAsked = true; + try + { + allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null); + } + catch (Exception) + { + allowPrivate = null; + } + } + + if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate)) + break; return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout or keep it out of the document's lists."; } } diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 9fae36b6..7c98927b 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -337,6 +337,63 @@ namespace GameCult.Caching.Tests Assert.That(back.Items.Single().Id, Is.EqualTo("000000000009")); } + // A consumer whose resolver composite ends in StandardResolverAllowPrivate (some models keep private [Key] members) gets + // that resolver's formatter for every keyed type. It writes the [Key] layout plus the private members, so it is no + // claim: the document encodes, the id is on disk, and the private value survives. + [Test] + public void AStandardResolverAllowPrivateFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApDoc), typeof(ApPlainDoc) }); + var doc = new ApDoc { Id = "a1", Items = { new ApArm { Id = "00000000000a", N = 4 }.WithSecret(11) } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApDoc), registry); + + var back = (ApDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ApDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("00000000000a")); + Assert.That(back.Items.Single().Secret, Is.EqualTo(11), "a private plain-value [Key] member is written"); + + var plain = new ApPlainDoc { Id = "a2", Items = { new ApPlainArm { Id = "00000000000b", N = 5 } } }; + var plainBytes = CultDocumentMessagePackSerialization.SerializeUntyped(plain, typeof(ApPlainDoc), registry); + Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); + } + + [MessagePackObject] + public sealed class ApArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + [Key(2)] private int secret; + [IgnoreMember] public int Secret => secret; + public ApArm WithSecret(int value) + { + secret = value; + return this; + } + } + + [CultDocument("tests.ap_doc", "tests.ap_doc.v1")] + [MessagePackObject] + public sealed class ApDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [MessagePackObject] + public sealed class ApPlainArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.ap_plain_doc", "tests.ap_plain_doc.v1")] + [MessagePackObject] + public sealed class ApPlainDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + [MessagePackObject] public sealed class LcArm { @@ -400,6 +457,9 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || + typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) + return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); return null; } From 8bbe56f555e5cef828599d407c6557a7bd46ff8d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:24:35 +0200 Subject: [PATCH 23/47] Contracts: the id walk follows declared types; keyed structs, every map interface and AllowPrivate named Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 13 ++++++++++--- .../Contracts/cultcache-store-composition.md | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index f1335e8f..2c83b102 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -408,8 +408,9 @@ fifth slot: ## Element Ids -Every object that sits in a list, array or dictionary value of a registered document -type, union subtypes included, carries an id: one string member with an integer `[Key]` +Every object that sits in a list, array or dictionary value (`IDictionary<,>` or +`IReadOnlyDictionary<,>`) of a registered document type, or of a keyed struct a document +holds, union subtypes included, carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The id identifies the element inside its list, so an override can name it (a path step is `[slot, elementId]`) and a base edit that reorders or inserts elements retargets nothing. @@ -432,7 +433,13 @@ elements retargets nothing. store; they carry ids and do not decide when a document carries them. - An object list's elements are written by their `[Key]` layout, because the id is a keyed member of the element. A list whose element type is not keyed - (`[MessagePackObject]` or `[Union]`) is a value list and is replaced whole. For an + (`[MessagePackObject]` or `[Union]`) is a value list and is replaced whole; so is a list + of keyed structs, since a struct is a value and never an element, but an object list + inside a keyed struct is walked like any other. The walk follows declared member types: + a member declared `object` is untyped on disk, and a subclass's extra members under a + non-union base-typed member are not written, so neither is walked. A non-public `[Key]` + member on the way to an object list is refused at registration, because MessagePack + writes it only under `AllowPrivate` and the element-id walk reads public members. For an object list, C# refuses at registration a concrete element type that is not keyed and `[MessagePackFormatter]` on any type, collection type or member on the way to the list or on the id member, and refuses at the first encode or decode of the document a formatter diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 545d9a8f..c8e2f17b 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -299,8 +299,9 @@ exclusive open of the same lock file exclude each other is not established. the walk passes (`List`, `T[]`, a collection class, a dictionary on the way). Those are written by their `[Key]` layout so each element's id reaches disk, and the codec refuses the document on first encode or decode, naming the resolver and the - type. A resolver that returns MessagePack's standard formatter for a type does not - claim it. A value list (`List`, a list of a non-keyed class) is not on the + type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a + type does not claim it (registration refuses a non-public `[Key]` member on the way to + an object list, so the private-member formatter writes every id). A value list (`List`, a list of a non-keyed class) is not on the path, so a resolver may claim it. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. From 70ffa2af9a201b5ab127c47d24fbdfc1a6a05ce1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:32:25 +0200 Subject: [PATCH 24/47] Element ids: Classify keeps only the string special case; tests pin nullable structs, unkeyed structs, inheritance and an AllowPrivate formatter that cannot be built Stryker found the primitive, enum and object tests in Classify redundant with the value-type and unkeyed branches, so they are gone. The new tests kill the survivors that were behaviour: a nullable keyed struct walked as its struct, an unannotated struct treated as a value, [MessagePackObject] reaching a subclass, and a claim on a type StandardResolverAllowPrivate cannot format. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultElementIds.cs | 7 ++-- .../ElementIdEngineTests.cs | 20 +++++++++ .../UnionArmGuardTests.cs | 41 +++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index fcbf8079..c27fef5f 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -567,8 +567,9 @@ namespace GameCult.Caching // The one recognition of what MessagePack writes for a declared type: registration, Plan, Holds and the codec's id path // all go through it. Inner is a map's value type, a sequence's element type, or the type itself for an owner. - // Leaf a value written whole that holds no element list: primitives, enums, strings, object (untyped on disk), - // a struct with no [MessagePackObject] or [Union], and a class with neither. + // Leaf a value written whole that holds no element list: strings (an IEnumerable MessagePack writes whole), + // a struct, class or interface with no [MessagePackObject] or [Union] (primitives, enums, float2, DateTime, + // and object, which is untyped on disk). // Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>. // Sequence an array or an IEnumerable. // Owner a class, interface or struct MessagePack serializes by key. A keyed struct is walked as an owner; it is @@ -576,7 +577,7 @@ namespace GameCult.Caching private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type => { type = Nullable.GetUnderlyingType(type) ?? type; - if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type == typeof(object)) + if (type == typeof(string)) return (Kind.Leaf, type); if (type.IsValueType) return IsKeyed(type) ? (Kind.Owner, type) : (Kind.Leaf, type); diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 4e9bb460..45ab621f 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -402,6 +402,21 @@ namespace GameCult.Caching.Tests Is.EquivalentTo(new[] { typeof(HStructValues), typeof(List), typeof(EIded) }), "a list of keyed structs is a value list"); var values = new HStructValues { Structs = { new SValue { N = 1 } }, Objects = { new EIded() } }; Assert.That(Assign(values, "r", deterministic: false), Is.EqualTo(1)); + + var nullable = new HNullableStructIded { Hold = new SHoldIded { Items = new List { new EIded() } } }; + Assert.That(Assign(nullable, "r", deterministic: false), Is.EqualTo(1), "a nullable struct is walked as its struct"); + Assert.That(CultElementIds.Holds(typeof(HNullableStructIded), nullable), Is.True); + } + + // A type is an owner or an element only when MessagePack serializes it by key ([MessagePackObject], inherited, or + // [Union]). A struct without either is a value however many [Key] members it carries. + [Test] + public void OnlyAKeyedTypeIsWalked() + { + Assert.That(Problems(typeof(HUnkeyedStruct)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HUnkeyedStruct)).IdPath, Is.Empty); + Assert.That(Problems(typeof(HInheritedBare)).Single(), Does.Contain("InheritedBare").And.Contain("[CultElementId]"), + "[MessagePackObject] reaches a subclass through inheritance"); } // Every dictionary interface MessagePack writes as a map is walked by one path, and the minted path text is the one a @@ -615,6 +630,11 @@ namespace GameCult.Caching.Tests [MessagePackObject] public struct SHold { [Key(0)] public List Items; } [MessagePackObject] public struct SHoldIded { [Key(0)] public List Items; } [MessagePackObject] public struct SValue { [Key(0)] public int N; } + public struct UnkeyedS { [Key(0)] public List Items; } + [MessagePackObject] public class HUnkeyedStruct { [Key(0)] public UnkeyedS Value; } + public class InheritedBare : EBare { } + [MessagePackObject] public class HInheritedBare { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HNullableStructIded { [Key(0)] public SHoldIded? Hold; } [MessagePackObject] public class HStruct { [Key(0)] public SHold Hold; } [MessagePackObject] public class HStructList { [Key(0)] public List Holds { get; set; } = new(); } [MessagePackObject] public class HNullableStruct { [Key(0)] public SHold? Hold; } diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 7c98927b..e6a11309 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -357,6 +357,46 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); } + // StandardResolverAllowPrivate cannot always build a formatter (a private readonly [Key] field has no constructor to + // fill it). That is not an escape from the claim check: a resolver formatting such a type is refused like any other. + [Test] + public void AFormatterThatStandardResolverAllowPrivateCannotBuildStillClaims() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApRoDoc) }); + var doc = new ApRoDoc { Id = "r1", Items = { new ApRoArm { Id = "00000000000c" } } }; + + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApRoDoc), registry))!; + + Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(ApRoArm))); + } + + [MessagePackObject] + public sealed class ApRoArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] private readonly int fixedValue = 3; + [IgnoreMember] public int FixedValue => fixedValue; + } + + [CultDocument("tests.ap_ro_doc", "tests.ap_ro_doc.v1")] + [MessagePackObject] + public sealed class ApRoDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + private sealed class ApRoFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, ApRoArm value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public ApRoArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new ApRoArm(); + } + } + [MessagePackObject] public sealed class ApArm { @@ -457,6 +497,7 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(ApRoArm)) return (IMessagePackFormatter)(object)new ApRoFormatter(); if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); From 8ddb8a03e19afc16e996038c4daa94e192da4a7c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:37:11 +0200 Subject: [PATCH 25/47] UnionArmGuardTests: the unbuildable-AllowPrivate claim test uses a private delegate member Co-Authored-By: Claude Sonnet 5.5 --- .../UnionArmGuardTests.cs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index e6a11309..eed82076 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -357,43 +357,44 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); } - // StandardResolverAllowPrivate cannot always build a formatter (a private readonly [Key] field has no constructor to - // fill it). That is not an escape from the claim check: a resolver formatting such a type is refused like any other. + // StandardResolverAllowPrivate cannot always build a formatter (a private [Key] delegate member has none). That is not an + // escape from the claim check: a resolver formatting such a type is refused like any other. [Test] public void AFormatterThatStandardResolverAllowPrivateCannotBuildStillClaims() { - var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApRoDoc) }); - var doc = new ApRoDoc { Id = "r1", Items = { new ApRoArm { Id = "00000000000c" } } }; + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApHookDoc) }); + var doc = new ApHookDoc { Id = "r1", Items = { new ApHookArm { Id = "00000000000c" } } }; - var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApRoDoc), registry))!; + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApHookDoc), registry))!; - Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(ApRoArm))); + Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(ApHookArm))); } [MessagePackObject] - public sealed class ApRoArm + public sealed class ApHookArm { [Key(0)] [CultElementId] public string Id = string.Empty; - [Key(1)] private readonly int fixedValue = 3; - [IgnoreMember] public int FixedValue => fixedValue; +#pragma warning disable CS0169, CS0649 + [Key(1)] private Action? hook; +#pragma warning restore CS0169, CS0649 } - [CultDocument("tests.ap_ro_doc", "tests.ap_ro_doc.v1")] + [CultDocument("tests.ap_hook_doc", "tests.ap_hook_doc.v1")] [MessagePackObject] - public sealed class ApRoDoc + public sealed class ApHookDoc { [Key(0)] [CultName] public string Id = string.Empty; - [Key(1)] public List Items = new(); + [Key(1)] public List Items = new(); } - private sealed class ApRoFormatter : IMessagePackFormatter + private sealed class ApHookFormatter : IMessagePackFormatter { - public void Serialize(ref MessagePackWriter writer, ApRoArm value, MessagePackSerializerOptions options) => writer.WriteNil(); + public void Serialize(ref MessagePackWriter writer, ApHookArm value, MessagePackSerializerOptions options) => writer.WriteNil(); - public ApRoArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + public ApHookArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); - return new ApRoArm(); + return new ApHookArm(); } } @@ -497,7 +498,7 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); - if (typeof(T) == typeof(ApRoArm)) return (IMessagePackFormatter)(object)new ApRoFormatter(); + if (typeof(T) == typeof(ApHookArm)) return (IMessagePackFormatter)(object)new ApHookFormatter(); if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); From 494ad9abfb2233748c0dc8aebfb65ac6a2aa605b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 01:39:17 +0200 Subject: [PATCH 26/47] UnionArmGuardTests: drop the unbuildable-AllowPrivate fixture No fixture makes StandardResolverAllowPrivate throw when asked for a formatter (Stryker: the catch stays NoCoverage), so the test pinned nothing the claim tests do not. Co-Authored-By: Claude Sonnet 5.5 --- .../UnionArmGuardTests.cs | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index eed82076..7c98927b 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -357,47 +357,6 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); } - // StandardResolverAllowPrivate cannot always build a formatter (a private [Key] delegate member has none). That is not an - // escape from the claim check: a resolver formatting such a type is refused like any other. - [Test] - public void AFormatterThatStandardResolverAllowPrivateCannotBuildStillClaims() - { - var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApHookDoc) }); - var doc = new ApHookDoc { Id = "r1", Items = { new ApHookArm { Id = "00000000000c" } } }; - - var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApHookDoc), registry))!; - - Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(ApHookArm))); - } - - [MessagePackObject] - public sealed class ApHookArm - { - [Key(0)] [CultElementId] public string Id = string.Empty; -#pragma warning disable CS0169, CS0649 - [Key(1)] private Action? hook; -#pragma warning restore CS0169, CS0649 - } - - [CultDocument("tests.ap_hook_doc", "tests.ap_hook_doc.v1")] - [MessagePackObject] - public sealed class ApHookDoc - { - [Key(0)] [CultName] public string Id = string.Empty; - [Key(1)] public List Items = new(); - } - - private sealed class ApHookFormatter : IMessagePackFormatter - { - public void Serialize(ref MessagePackWriter writer, ApHookArm value, MessagePackSerializerOptions options) => writer.WriteNil(); - - public ApHookArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - reader.Skip(); - return new ApHookArm(); - } - } - [MessagePackObject] public sealed class ApArm { @@ -498,7 +457,6 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); - if (typeof(T) == typeof(ApHookArm)) return (IMessagePackFormatter)(object)new ApHookFormatter(); if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); From 9fd5ae1f0973da3dd7e4226f28ca2e1a74e56e4d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:49:18 +0200 Subject: [PATCH 27/47] CultElementIds: a keyed type is read by key before it is read as a sequence Classify tested IEnumerable before [MessagePackObject], so a keyed class that enumerates itself recursed until the stack overflowed during registration (s4, keyed-enumerable-misclassified), and a keyed class that merely enumerates had its keyed lists neither judged nor minted. Keyed first: MessagePack writes both by key. VisitMemberType also stops at a type already on its own through chain, which ends an unkeyed Node : IEnumerable. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultElementIds.cs | 16 ++++-- .../ElementIdEngineTests.cs | 57 +++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index c27fef5f..77ba8501 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -148,6 +148,10 @@ namespace GameCult.Caching private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, bool hidden, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; + // A sequence or map whose element is itself (an unkeyed Node : IEnumerable) ends the type walk here; without + // this it would recurse until the stack overflows. + if (Array.IndexOf(through, memberType) >= 0) + return; var (kind, inner) = Classify(memberType); switch (kind) { @@ -565,8 +569,10 @@ namespace GameCult.Caching private static readonly ConcurrentDictionary Classes = new(); - // The one recognition of what MessagePack writes for a declared type: registration, Plan, Holds and the codec's id path - // all go through it. Inner is a map's value type, a sequence's element type, or the type itself for an owner. + // The walk's model of a declared type: registration, Plan, Holds and the codec's id path all go through it. A + // [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable; the codec checks, for every type + // on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not. + // Inner is a map's value type, a sequence's element type, or the type itself for an owner. // Leaf a value written whole that holds no element list: strings (an IEnumerable MessagePack writes whole), // a struct, class or interface with no [MessagePackObject] or [Union] (primitives, enums, float2, DateTime, // and object, which is untyped on disk). @@ -579,13 +585,15 @@ namespace GameCult.Caching type = Nullable.GetUnderlyingType(type) ?? type; if (type == typeof(string)) return (Kind.Leaf, type); + if (IsKeyed(type)) + return (Kind.Owner, type); if (type.IsValueType) - return IsKeyed(type) ? (Kind.Owner, type) : (Kind.Leaf, type); + return (Kind.Leaf, type); if (MapTypes(type) is { } map) return (Kind.Map, map.Value); if (ElementTypeOf(type) is { } element) return (Kind.Sequence, element); - return IsKeyed(type) ? (Kind.Owner, type) : (Kind.Leaf, type); + return (Kind.Leaf, type); }); // A type MessagePack serializes by key: [MessagePackObject] or [Union]. diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 45ab621f..1a4417b8 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -485,7 +485,64 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Inspect(typeof(HHiddenList)).IdPath, Is.Empty); } + // MessagePack writes a [MessagePackObject] class that is only IEnumerable by its keys, so the walk reads it by key. A + // keyed tree node that enumerates its own children is the shape that overflowed the stack when enumeration came first. + [Test] + public void ATypeThatEnumeratesItselfIsWalkedByKeyAndTerminates() + { + var survey = CultElementIds.Inspect(typeof(HKNode)); + Assert.That(survey.Problems, Is.Empty); + Assert.That(survey.IdPath, Is.EquivalentTo(new[] { typeof(HKNode), typeof(List), typeof(KNode) })); + + var root = new HKNode { Nodes = { new KNode { Children = { new KNode { Children = { new KNode() } } } } } }; + Assert.That(Assign(root, "r", deterministic: true), Is.EqualTo(3)); + Assert.That(root.Nodes[0].Id, Has.Length.EqualTo(12)); + Assert.That(root.Nodes[0].Children[0].Children[0].Id, Has.Length.EqualTo(12)); + + // An unkeyed type that enumerates itself holds no element list and ends the type walk. + var unkeyed = CultElementIds.Inspect(typeof(HUNode)); + Assert.That(unkeyed.Problems, Is.Empty); + Assert.That(unkeyed.IdPath, Is.Empty); + + // A keyed class that only enumerates is read by its keyed lists, so both are judged and minted. + Assert.That(Problems(typeof(KBag)).Single(), Does.Contain("EBare")); + var bag = new KBagIded { Items = { new EIded(), new EIded() } }; + Assert.That(Assign(bag, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True); + } + // ---- shapes ---- + [MessagePackObject] + public class KNode : IEnumerable + { + [Key(0)] [CultElementId] public string? Id { get; set; } + [Key(1)] public List Children { get; set; } = new(); + public IEnumerator GetEnumerator() => Children.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HKNode { [Key(0)] public List Nodes { get; set; } = new(); } + public class UNode : IEnumerable + { + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HUNode { [Key(0)] public UNode? N { get; set; } } + [MessagePackObject] + public class KBag : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + [Key(1)] public List Other { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] + public class KBagIded : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } [MessagePackObject] public class ENest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } From ee3e2620644ff1a4bb2af3c493d5b905b981ca53 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:51:24 +0200 Subject: [PATCH 28/47] CultElementIds: the walk reads the members MessagePack writes, and an id it cannot read back is refused MembersOf is now MessagePack's StandardResolver write set: public fields and properties with a public getter, none of them [IgnoreMember]. UnreadKeyedMembers (renamed from NonPublicKeyedMembers) is every other [Key] member, including a public property whose getter is private. IdProblem takes id candidates from every instance member up the base chain, refuses an id MessagePack does not write, and refuses a property id with a non-public setter, which MessagePack writes but does not read back (s4: walk-member-set-differs, private-setter-id-dropped-on-read). An oracle test pins the member set to MessagePack's ObjectSerializationInfo. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultElementIds.cs | 52 +++++- .../ElementIdEngineTests.cs | 158 ++++++++++++++++++ 2 files changed, 201 insertions(+), 9 deletions(-) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 77ba8501..34541f34 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -136,7 +136,7 @@ namespace GameCult.Caching return; foreach (var (member, memberType) in KeyedMembers(type)) VisitMemberType(documentType, type, member, memberType, Array.Empty(), false, seen, survey, graph); - foreach (var (member, memberType) in NonPublicKeyedMembers(type)) + foreach (var (member, memberType) in UnreadKeyedMembers(type)) VisitMemberType(documentType, type, member, memberType, Array.Empty(), true, seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) { @@ -276,7 +276,10 @@ namespace GameCult.Caching private static string? IdProblem(Type element) { - var ids = MembersOf(element).Where(member => Declares(member, typeof(CultElementIdAttribute))).ToArray(); + // Candidates come from every instance member up the base chain, public or not, so an id MessagePack does not write + // is named as such instead of being reported missing. An override or a hiding member is one member, the most derived. + var ids = AllInstanceMembers(element).Where(member => Declares(member, typeof(CultElementIdAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()).ToArray(); if (ids.Length == 0) return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity."; if (ids.Length > 1) @@ -288,11 +291,16 @@ namespace GameCult.Caching return $"declares [CultElementId] on {id.Name}, which is not a string."; if (KeyOf(id) is not { } idKey) return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted."; + if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written) + return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], non-public, or a property whose getter is not public); the id would never reach disk."; + id = written; var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey); if (sharing.Member != null) return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own."; if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true }) return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it."; + if (id is PropertyInfo { SetMethod.IsPublic: false }) + return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine)."; var source = id.GetCustomAttribute(true)!.DerivedFrom; if (source != null) { @@ -533,9 +541,33 @@ namespace GameCult.Caching return shape; } + // The members MessagePack's StandardResolver writes for an integer-keyed type: public instance fields, and public + // instance properties with a public getter, none of them [IgnoreMember]. TheWalkReadsTheMembersMessagePackWrites checks + // this set against MessagePack's own member discovery. private static IEnumerable MembersOf(Type type) => type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast() - .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod != null)); + .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + .Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true })) + .Where(member => !Declares(member, typeof(IgnoreMemberAttribute))); + + // Every instance field and indexer-free property declared on the type or any base, most derived first. + private static IEnumerable AllInstanceMembers(Type type) + { + const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + for (var level = type; level != null && level != typeof(object); level = level.BaseType) + { + foreach (var field in level.GetFields(flags)) + yield return field; + foreach (var property in level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0)) + yield return property; + } + } + + // The members the walk reads, as MessagePack writes them, for the oracle test that checks both against MessagePack. + internal static IEnumerable WrittenKeyedMembers(Type type) => KeyedMembers(type).Select(entry => entry.Member); + + // The [Key] members MessagePack writes only under AllowPrivate, for the same test. + internal static IEnumerable UnreadMembers(Type type) => UnreadKeyedMembers(type).Select(entry => entry.Member); private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member))); @@ -544,16 +576,18 @@ namespace GameCult.Caching // override of a base virtual member would not see its base's attribute. private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); - // [Key] members MessagePack writes only under AllowPrivate: non-public fields and properties, up the base chain. - private static IEnumerable<(MemberInfo Member, Type Type)> NonPublicKeyedMembers(Type type) + // [Key] members MessagePack writes only under AllowPrivate: non-public fields and properties, and public properties whose + // getter is not public, up the base chain. An [IgnoreMember] member is never written, so it is not one of them. + private static IEnumerable<(MemberInfo Member, Type Type)> UnreadKeyedMembers(Type type) { for (var level = type; level != null && level != typeof(object); level = level.BaseType) { - const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; - foreach (var member in level.GetFields(flags).Cast() - .Concat(level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0))) + const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + foreach (var member in level.GetFields(flags).Where(field => !field.IsPublic).Cast() + .Concat(level.GetProperties(flags).Where(property => + property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: false }))) { - if (KeyOf(member) != null) + if (KeyOf(member) != null && !Declares(member, typeof(IgnoreMemberAttribute))) yield return (member, TypeOf(member)); } } diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 1a4417b8..3f6b084e 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -511,7 +511,165 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True); } + // The walk reads the members MessagePack writes: a public field or a property with a public getter, no [IgnoreMember]. + // Anything else that carries a [Key] is MessagePack's AllowPrivate business and, on the way to an object list, is refused. + [Test] + public void TheWalkReadsTheMembersMessagePackWrites() + { + Assert.That(Problems(typeof(PubIgn)), Is.Empty, "an ignored member is never written, so its list is nobody's business"); + Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not a hidden list"); + var held = Problems(typeof(PgHold)).Single(); + Assert.That(held, Does.Contain("Items").And.Contain("non-public")); + + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix) }) + { + var written = MessagePackMembers(type, allowPrivate: false); + Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), Is.EquivalentTo(written), type.Name); + var readable = MessagePackMembers(type, allowPrivate: true); + Assert.That(CultElementIds.UnreadMembers(type).Select(member => member.Name), Is.EquivalentTo(readable.Except(written)), type.Name); + } + + Assert.That(CultElementIds.UnreadMembers(typeof(MemberMix)).Select(member => member.Name), Is.EquivalentTo(new[] { "priv" })); + Assert.That(CultElementIds.UnreadMembers(typeof(PgHold)).Select(member => member.Name), Is.EquivalentTo(new[] { "Items" })); + } + + // The names of the members MessagePack's own discovery reads for a type, by reflection into its internal type: the + // oracle the walk's member set is pinned to. + private static List MessagePackMembers(Type type, bool allowPrivate) + { + const string moved = "MessagePack moved its member discovery; re-derive MembersOf"; + const System.Reflection.BindingFlags all = System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic; + var info = typeof(MessagePackSerializer).Assembly.GetType("MessagePack.Internal.ObjectSerializationInfo"); + var create = info?.GetMethod("CreateOrNull", all); + if (create == null || create.GetParameters().Length != 4) + { + Assert.Fail(moved); + return new List(); + } + + var result = create.Invoke(null, new object[] { type, false, false, allowPrivate }); + var members = result?.GetType().GetProperty("Members", all)?.GetValue(result) as System.Collections.IEnumerable; + if (members == null) + { + Assert.Fail(moved); + return new List(); + } + + var names = new List(); + foreach (var member in members) + { + var readable = member.GetType().GetProperty("IsReadable", all)?.GetValue(member); + var name = member.GetType().GetProperty("Name", all)?.GetValue(member) as string; + if (readable is not bool || name == null) + Assert.Fail(moved); + if ((bool)readable!) + names.Add(name!); + } + + return names; + } + + // An id MessagePack writes but cannot read back would be re-minted differently on every load; one it does not write + // never reaches disk. + [Test] + public void AnIdMessagePackCannotReadBackIsRefused() + { + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id")); + Assert.That(Problems(typeof(HIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HPrivateId)).Single(), Does.Contain("does not write").And.Contain("id")); + Assert.That(Problems(typeof(HInit)), Is.Empty); + Assert.That(Problems(typeof(HCtorId)), Is.Empty); + + foreach (var holder in new object[] + { + new HInit { Items = { new EInit(), new EInit() } }, + new HCtorId { Items = { new ECtorId(null), new ECtorId(null) } }, + }) + { + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + var bytes = MessagePackSerializer.Serialize(holder.GetType(), holder, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + var back = MessagePackSerializer.Deserialize(holder.GetType(), bytes, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + Assert.That(Ids(back), Is.EqualTo(Ids(holder)), holder.GetType().Name); + Assert.That(Ids(back), Has.All.Match("^[0-9a-f]{12}$"), holder.GetType().Name); + } + } + + private static string?[] Ids(object holder) => holder switch + { + HInit init => init.Items.Select(item => item.Id).ToArray(), + HCtorId ctor => ctor.Items.Select(item => item.Id).ToArray(), + _ => throw new ArgumentException(holder.GetType().Name), + }; + + // A subclass that redeclares the id member does not change which member the declared element writes. + [Test] + public void AnElementIsReadByItsDeclaredTypeNotItsRuntimeType() + { + Assert.That(Problems(typeof(HDoc2)), Is.Empty); + static HE Hd(string? declared = null, string? redeclared = null) + { + var item = new HD { Id = redeclared }; + ((HE)item).Id = declared; + return item; + } + + var minted = new HDoc2 { Items = { Hd() } }; + Assert.That(Assign(minted, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(minted.Items[0].Id, Does.Match("^[0-9a-f]{12}$")); + Assert.That(((HD)minted.Items[0]).Id, Is.Null, "the redeclared member is never written, so it is not minted"); + + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "not-hex") } }, "r", false), + Throws.TypeOf(), "the declared member is checked"); + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "not-hex") } }, "r", false), + Throws.Nothing, "the redeclared member is never read"); + + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "0123456789ab") } }), Is.True); + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "0123456789ab") } }), Is.False); + } + // ---- shapes ---- + [MessagePackObject] public class PubIgn { [Key(0)] [IgnoreMember] public List L { get; set; } = new(); } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class ApIgn { [Key(0)] [IgnoreMember] private List l = new(); } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public class PgHold { [Key(0)] public List Items { private get; set; } = new(); } + [MessagePackObject] + public class MemberMix + { + [Key(0)] public int A; + [Key(1)] [IgnoreMember] public List? Ign; + [Key(2)] public string? PS { get; private set; } + [Key(3)] public string? GetOnly { get; } +#pragma warning disable CS0169, CS0414, CS0649 + [Key(4)] private int priv; +#pragma warning restore CS0169, CS0414, CS0649 + [Key(5)] public string? InitOnly { get; init; } + [IgnoreMember] public int Priv => priv; + } + + [MessagePackObject] public class EPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EIgnId { [Key(0)] [IgnoreMember] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HIgnId { [Key(0)] public List Items { get; set; } = new(); } + #pragma warning disable CS0169, CS0649 + [MessagePackObject] public class EPrivateId { [Key(0)] [CultElementId] private string? id; } +#pragma warning restore CS0169, CS0649 + [MessagePackObject] public class HPrivateId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EInit { [Key(0)] [CultElementId] public string? Id { get; init; } } + [MessagePackObject] public class HInit { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] + public class ECtorId + { + [SerializationConstructor] public ECtorId(string? id) { Id = id; } + [Key(0)] [CultElementId] public string? Id { get; set; } + } + [MessagePackObject] public class HCtorId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HE { [Key(0)] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HD : HE { [Key(2)] [CultElementId] public new string? Id { get; set; } } + [MessagePackObject] public class HDoc2 { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class KNode : IEnumerable { From 5157c647afc3c5a048d222ce582c913feb35015c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:53:28 +0200 Subject: [PATCH 29/47] Codec: confirm MessagePack writes every id-path type the way the walk reads it CultElementIds is the walk's model of a type; MessagePack's StandardResolver owns how a type is written, and nothing checked that the two agree. ResolverClaim now asks, for each id-path type, whether StandardResolver's formatter is the object or union resolver's (a type read by key) or a collection resolver's (a collection), and refuses the document at first encode or decode when it is not, naming the type. A keyed type that enumerates objects goes on the path so a keyed List subclass is caught. A type MessagePack cannot build a formatter for, and a resolver ahead that throws, are typed refusals; the AllowPrivate catch is pinned (s4: keyed-enumerable-misclassified, allowprivate-catch-unpinned). Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 55 ++++++++- src/GameCult.Caching/CultElementIds.cs | 16 +++ .../UnionArmGuardTests.cs | 107 ++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index ec0326fb..8faab41e 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -8,6 +8,7 @@ using System.Reflection; using GameCult.Caching; using MessagePack; using MessagePack.Formatters; +using MessagePack.ImmutableCollection; using MessagePack.Resolvers; namespace GameCult.Caching.MessagePack; @@ -86,12 +87,36 @@ public static class CultDocumentMessagePackSerialization // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout // plus non-public [Key] members, and registration refuses a non-public member on the way to an object list, so it // claims nothing either. - var standard = generic.Invoke(StandardResolver.Instance, null); + object? standard; + try + { + standard = generic.Invoke(StandardResolver.Instance, null); + } + catch (Exception exception) + { + return $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message})."; + } + + if (!WrittenAsWalked(type, generic, standard)) + { + var owner = CultElementIds.WalksAsOwner(type); + return $"Cult document {documentType.Name}: MessagePack writes {type.Name} {(owner ? "as a collection" : "by its [Key] layout")}, but the element-id walk reads it {(owner ? "by its [Key] layout" : "as a collection")}, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other."; + } + var allowPrivateAsked = false; object? allowPrivate = null; foreach (var resolver in ahead) { - var formatter = generic.Invoke(resolver, null); + object? formatter; + try + { + formatter = generic.Invoke(resolver, null); + } + catch (Exception exception) + { + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type."; + } + if (formatter == null) continue; if (ReferenceEquals(formatter, standard)) @@ -118,6 +143,32 @@ public static class CultDocumentMessagePackSerialization return null; } + // The walk's kind for a type on the id path, checked against the resolver MessagePack actually delegates to. The public + // singletons hand out the same cached formatter instances StandardResolver does, so a reference match names the family. A + // type the walk reads by key must be written by the object or union resolver, and one it reads as a collection by a + // collection resolver; a keyed class that is also an ICollection is written as a collection and its [Key] layout dropped. + private static bool WrittenAsWalked(Type type, MethodInfo generic, object? standard) + { + if (standard == null) + return true; + var families = CultElementIds.WalksAsOwner(type) + ? new IFormatterResolver[] { DynamicUnionResolver.Instance, DynamicObjectResolver.Instance } + : new IFormatterResolver[] { DynamicGenericResolver.Instance, ImmutableCollectionResolver.Instance, BuiltinResolver.Instance }; + return families.Any(family => ReferenceEquals(Formatter(family, generic), standard)); + } + + private static object? Formatter(IFormatterResolver resolver, MethodInfo generic) + { + try + { + return generic.Invoke(resolver, null); + } + catch (Exception) + { + return null; + } + } + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers) { // The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 34541f34..eb6fb815 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -113,6 +113,9 @@ namespace GameCult.Caching // and the walk reads public members only. public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through, bool Hidden)> Edges { get; } = new(); public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through, bool Hidden)> Lists { get; } = new(); + + // Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too. + public List Enumerating { get; } = new(); } private static readonly ConcurrentDictionary IdCapable = new(); @@ -134,6 +137,8 @@ namespace GameCult.Caching { if (!seen.Add(type)) return; + if (EnumeratesObjects(type)) + graph.Enumerating.Add(type); foreach (var (member, memberType) in KeyedMembers(type)) VisitMemberType(documentType, type, member, memberType, Array.Empty(), false, seen, survey, graph); foreach (var (member, memberType) in UnreadKeyedMembers(type)) @@ -229,6 +234,11 @@ namespace GameCult.Caching Add(concrete); } + // A keyed type the walk reads by key but whose enumeration holds objects is on the path: the codec asks whether + // MessagePack writes it by key too. + foreach (var type in graph.Enumerating) + Add(type); + foreach (var type in path) { survey.IdPath.Add(type); @@ -593,6 +603,12 @@ namespace GameCult.Caching } } + // True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it. + internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner; + + private static bool EnumeratesObjects(Type type) => + (ElementTypeOf(type) ?? MapTypes(type)?.Value) is { } inner && Classify(inner).Kind != Kind.Leaf; + private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType; diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 7c98927b..3735239f 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -357,6 +357,111 @@ namespace GameCult.Caching.Tests Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); } + // The walk reads a [MessagePackObject] type by its keys, but MessagePack writes one that is also an ICollection as a + // collection and drops the keys. The codec compares the two for every type on the id path and refuses the document. + [Test] + public void AKeyedTypeMessagePackWritesAsACollectionIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(KCollDoc), typeof(KBagDoc) }); + Assert.That(CultElementIds.Inspect(typeof(KCollDoc)).IdPath, Does.Contain(typeof(KColl))); + + var coll = new KCollDoc { Id = "k1" }; + coll.Items.Add(new FbArm { Id = "00000000000c" }); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(coll, typeof(KCollDoc), registry))!; + Assert.That(thrown.Message, Does.Contain(nameof(KColl)).And.Contain("as a collection")); + + var bag = new KBagDoc { Id = "b1", Bag = new KBagE { Items = { new FbArm { Id = "00000000000d" } } } }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(bag, typeof(KBagDoc), registry); + var back = (KBagDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(KBagDoc), bytes, registry); + Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d")); + } + + // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. + [Test] + public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter() + { + foreach (var holder in new[] + { + typeof(G>), typeof(G), typeof(G), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G), typeof(G), typeof(G), + }) + { + if (holder != typeof(G)) + Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.Name); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + } + + // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter + // the claim check asks about (the claim stands) or a resolver ahead (it fails to build). + [Test] + public void AResolverThatThrowsIsATypedRefusal() + { + var first = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc)))!; + Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl))); + var second = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!; + Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build")); + } + + [MessagePackObject] + public sealed class KColl : List + { + [Key(0)] public int N; + } + + [CultDocument("tests.kcoll_doc", "tests.kcoll_doc.v1")] + [MessagePackObject] + public sealed class KCollDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KColl Items = new(); + } + + [MessagePackObject] + public sealed class KBagE : IEnumerable + { + [Key(0)] public List Items = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [CultDocument("tests.kbag_doc", "tests.kbag_doc.v1")] + [MessagePackObject] + public sealed class KBagDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KBagE Bag = new(); + } + + [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } + [Union(0, typeof(GuArm))] public interface IGu { } + [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [MessagePackObject] public struct GStruct { [Key(0)] public List Items; } + +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } + [MessagePackObject] public sealed class DkEl2 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } +#pragma warning restore CS0169, CS0414, CS0649 + [CultDocument("tests.dk_doc", "tests.dk_doc.v1")] + [MessagePackObject] + public sealed class DkDoc { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [CultDocument("tests.dk_doc2", "tests.dk_doc2.v1")] + [MessagePackObject] + public sealed class DkDoc2 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + + public sealed class DkFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N); + public DkEl Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => new DkEl { N = reader.ReadInt32() }; + } + [MessagePackObject] public sealed class ApArm { @@ -457,6 +562,8 @@ namespace GameCult.Caching.Tests if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(DkEl)) return (IMessagePackFormatter)(object)new DkFormatter(); + if (typeof(T) == typeof(DkEl2)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); From 7d4224bc96f176b6489731dd9be83169c4273e52 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:54:25 +0200 Subject: [PATCH 30/47] UnionArmGuardTests: name the collection shape whose id path is empty Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 3735239f..66596b78 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -394,7 +394,7 @@ namespace GameCult.Caching.Tests }) { if (holder != typeof(G)) - Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.Name); + Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.ToString()); Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); } } From 1956145ea2a9756d44c01fc1863192af6d92bef4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:56:18 +0200 Subject: [PATCH 31/47] UnionArmGuardTests: maps in the shape sweep hold a list-carrying object Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 66596b78..49754af0 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -387,10 +387,10 @@ namespace GameCult.Caching.Tests typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>>), - typeof(G>), typeof(G>), typeof(G>), - typeof(G>), typeof(G>), - typeof(G>), typeof(G>), - typeof(G>), typeof(G), typeof(G), typeof(G), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G), typeof(G), typeof(G), }) { if (holder != typeof(G)) @@ -443,6 +443,7 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } [Union(0, typeof(GuArm))] public interface IGu { } [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [MessagePackObject] public sealed class GHold { [Key(0)] public List Items = new(); } [MessagePackObject] public struct GStruct { [Key(0)] public List Items; } #pragma warning disable CS0169, CS0414, CS0649 From 744f2f78b1d9411cebd9f1bc0138d6360674b5e7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 02:58:10 +0200 Subject: [PATCH 32/47] Contracts: the written-member rule, readable ids and the codec's family check Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 14 +++++++++++--- .../Contracts/cultcache-store-composition.md | 6 ++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 2c83b102..966b2a1f 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -437,9 +437,17 @@ elements retargets nothing. of keyed structs, since a struct is a value and never an element, but an object list inside a keyed struct is walked like any other. The walk follows declared member types: a member declared `object` is untyped on disk, and a subclass's extra members under a - non-union base-typed member are not written, so neither is walked. A non-public `[Key]` - member on the way to an object list is refused at registration, because MessagePack - writes it only under `AllowPrivate` and the element-id walk reads public members. For an + non-union base-typed member are not written, so neither is walked. The walk reads the + members MessagePack writes: an integer `[Key]`, no `[IgnoreMember]`, a public field or a + property whose getter is public. Any other `[Key]` member on the way to an object list + (non-public, or a property with a non-public getter) is refused at registration, because + MessagePack writes it only under `AllowPrivate` and the element-id walk reads public + members. An id member needs a public setter or `init`, so the id MessagePack writes is read + back; one with a non-public setter, or one MessagePack does not write, is refused. A + `[MessagePackObject]` type is read by key even when it is `IEnumerable`; MessagePack + writes a keyed type that is also a collection (`ICollection`, `IList`, `IDictionary`) + as a collection, so C# refuses such a type at the first encode or decode when it is on the + path to an object list. For an object list, C# refuses at registration a concrete element type that is not keyed and `[MessagePackFormatter]` on any type, collection type or member on the way to the list or on the id member, and refuses at the first encode or decode of the document a formatter diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index c8e2f17b..f2990a3a 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -303,6 +303,12 @@ exclusive open of the same lock file exclude each other is not established. type does not claim it (registration refuses a non-public `[Key]` member on the way to an object list, so the private-member formatter writes every id). A value list (`List`, a list of a non-keyed class) is not on the path, so a resolver may claim it. +- The codec also checks, for every type on that path, that `StandardResolver` writes it + the way the walk reads it. A type read by key must be written by the object or union + resolver; a collection must be written by a collection resolver (generic, immutable or + builtin). Otherwise the codec refuses the document on first encode or decode, naming the + type. It also refuses, typed, a type MessagePack cannot build a formatter for, and a + resolver that throws while asked about one. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. From 956c077a1a273d57724e923bd5933f88ef90462a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 03:00:58 +0200 Subject: [PATCH 33/47] UnionArmGuardTests: pin the refusal for a type MessagePack cannot build a formatter for Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 49754af0..37f51cd8 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -408,6 +408,10 @@ namespace GameCult.Caching.Tests Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl))); var second = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!; Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build")); + + // A type MessagePack itself cannot build a formatter for (two public members on one key) is refused, naming the type. + var third = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc3)))!; + Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter")); } [MessagePackObject] @@ -450,6 +454,10 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class DkEl { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } [MessagePackObject] public sealed class DkEl2 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } #pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl3 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] public int M; } + [CultDocument("tests.dk_doc3", "tests.dk_doc3.v1")] + [MessagePackObject] + public sealed class DkDoc3 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } [CultDocument("tests.dk_doc", "tests.dk_doc.v1")] [MessagePackObject] public sealed class DkDoc { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } From 48d3f667e6bd5e7467ececd5516f9fd673427ab4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 03:09:39 +0200 Subject: [PATCH 34/47] Element ids: a keyed map counts as enumerating objects; family check tests assert the exact refusal EnumeratesObjects took ElementTypeOf first, which for a dictionary is the KeyValuePair (a leaf), so a keyed Dictionary subclass never reached the path; the map's value type is asked first. The unreachable null-standard guard in WrittenAsWalked is removed. Tests pin the direction and wording of the refusal, the attribute-formatter collection case and the unwrapped inner exception. Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 2 - src/GameCult.Caching/CultElementIds.cs | 2 +- .../UnionArmGuardTests.cs | 40 +++++++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 8faab41e..379d3e36 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -149,8 +149,6 @@ public static class CultDocumentMessagePackSerialization // collection resolver; a keyed class that is also an ICollection is written as a collection and its [Key] layout dropped. private static bool WrittenAsWalked(Type type, MethodInfo generic, object? standard) { - if (standard == null) - return true; var families = CultElementIds.WalksAsOwner(type) ? new IFormatterResolver[] { DynamicUnionResolver.Instance, DynamicObjectResolver.Instance } : new IFormatterResolver[] { DynamicGenericResolver.Instance, ImmutableCollectionResolver.Instance, BuiltinResolver.Instance }; diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index eb6fb815..4746b59c 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -607,7 +607,7 @@ namespace GameCult.Caching internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner; private static bool EnumeratesObjects(Type type) => - (ElementTypeOf(type) ?? MapTypes(type)?.Value) is { } inner && Classify(inner).Kind != Kind.Leaf; + (MapTypes(type)?.Value ?? ElementTypeOf(type)) is { } inner && Classify(inner).Kind != Kind.Leaf; private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 37f51cd8..91c94d3c 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -368,7 +368,11 @@ namespace GameCult.Caching.Tests var coll = new KCollDoc { Id = "k1" }; coll.Items.Add(new FbArm { Id = "00000000000c" }); var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(coll, typeof(KCollDoc), registry))!; - Assert.That(thrown.Message, Does.Contain(nameof(KColl)).And.Contain("as a collection")); + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KColl)} as a collection, but the element-id walk reads it by its [Key] layout")); + + // A keyed map is read by key too, so a keyed Dictionary subclass is on the path and refused the same way. + var map = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(KDictDoc)))!; + Assert.That(map.Message, Does.Contain($"writes {nameof(KDict)} as a collection")); var bag = new KBagDoc { Id = "b1", Bag = new KBagE { Items = { new FbArm { Id = "00000000000d" } } } }; var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(bag, typeof(KBagDoc), registry); @@ -399,6 +403,14 @@ namespace GameCult.Caching.Tests } } + // The walk reads a List subclass as a collection; a type MessagePack writes by an attribute formatter instead is a mismatch. + [Test] + public void ACollectionMessagePackWritesByAnotherFormatterIsRefused() + { + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(FormattedRoles)} by its [Key] layout, but the element-id walk reads it as a collection")); + } + // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter // the claim check asks about (the claim stands) or a resolver ahead (it fails to build). [Test] @@ -407,11 +419,12 @@ namespace GameCult.Caching.Tests var first = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc)))!; Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl))); var second = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!; - Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build")); + Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build").And.Not.Contain("target of an invocation"), + "the resolver's own exception is named, not the reflection wrapper"); // A type MessagePack itself cannot build a formatter for (two public members on one key) is refused, naming the type. var third = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc3)))!; - Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter")); + Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter").And.Not.Contain("target of an invocation")); } [MessagePackObject] @@ -428,6 +441,20 @@ namespace GameCult.Caching.Tests [Key(1)] public KColl Items = new(); } + [MessagePackObject] + public sealed class KDict : Dictionary + { + [Key(0)] public int N; + } + + [CultDocument("tests.kdict_doc", "tests.kdict_doc.v1")] + [MessagePackObject] + public sealed class KDictDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KDict Items = new(); + } + [MessagePackObject] public sealed class KBagE : IEnumerable { @@ -465,6 +492,13 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class DkDoc2 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [MessagePackFormatter(typeof(FormattedRolesFormatter))] public sealed class FormattedRoles : List { } + public sealed class FormattedRolesFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FormattedRoles value, MessagePackSerializerOptions options) => writer.WriteNil(); + public FormattedRoles Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new FormattedRoles(); } + } + public sealed class DkFormatter : IMessagePackFormatter { public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N); From 127432efb1c00b858ecb261110cdc959e2e6628e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 03:29:25 +0200 Subject: [PATCH 35/47] UnionArmGuardTests: a non-generic list that enumerates arms joins the shape sweep Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 91c94d3c..d67a86d5 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -394,7 +394,7 @@ namespace GameCult.Caching.Tests typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), - typeof(G>), typeof(G), typeof(G), typeof(G), + typeof(G>), typeof(G), typeof(G), typeof(G), typeof(G), }) { if (holder != typeof(G)) @@ -471,6 +471,12 @@ namespace GameCult.Caching.Tests [Key(1)] public KBagE Bag = new(); } + // A non-generic list that also enumerates arms: the one collection shape MessagePack's BuiltinResolver writes. + public sealed class ArrayOfArms : System.Collections.ArrayList, IEnumerable + { + IEnumerator IEnumerable.GetEnumerator() => this.Cast().GetEnumerator(); + } + [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } [Union(0, typeof(GuArm))] public interface IGu { } [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } From fb2be32b60501cfc70a9a61e292d7eaf0e98275f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 03:35:30 +0200 Subject: [PATCH 36/47] Revert "UnionArmGuardTests: a non-generic list that enumerates arms joins the shape sweep" This reverts commit 127432efb1c00b858ecb261110cdc959e2e6628e. --- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index d67a86d5..91c94d3c 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -394,7 +394,7 @@ namespace GameCult.Caching.Tests typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), - typeof(G>), typeof(G), typeof(G), typeof(G), typeof(G), + typeof(G>), typeof(G), typeof(G), typeof(G), }) { if (holder != typeof(G)) @@ -471,12 +471,6 @@ namespace GameCult.Caching.Tests [Key(1)] public KBagE Bag = new(); } - // A non-generic list that also enumerates arms: the one collection shape MessagePack's BuiltinResolver writes. - public sealed class ArrayOfArms : System.Collections.ArrayList, IEnumerable - { - IEnumerator IEnumerable.GetEnumerator() => this.Cast().GetEnumerator(); - } - [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } [Union(0, typeof(GuArm))] public interface IGu { } [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } From 6e70626bf92bfc1102494f2ce60971416a49742f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 03:36:15 +0200 Subject: [PATCH 37/47] ElementIdEngineTests: the keyed self-enumerating node round-trips through MessagePack with its ids Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/ElementIdEngineTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 3f6b084e..29fc497b 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -498,6 +498,10 @@ namespace GameCult.Caching.Tests Assert.That(Assign(root, "r", deterministic: true), Is.EqualTo(3)); Assert.That(root.Nodes[0].Id, Has.Length.EqualTo(12)); Assert.That(root.Nodes[0].Children[0].Children[0].Id, Has.Length.EqualTo(12)); + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(root, options), options); + Assert.That(back.Nodes[0].Id, Is.EqualTo(root.Nodes[0].Id), "MessagePack writes the node by key, ids included"); + Assert.That(back.Nodes[0].Children[0].Children[0].Id, Is.EqualTo(root.Nodes[0].Children[0].Children[0].Id)); // An unkeyed type that enumerates itself holds no element list and ends the type walk. var unkeyed = CultElementIds.Inspect(typeof(HUNode)); From a5241112ded76eb102035821924fb03b8274595e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 20:52:13 +0200 Subject: [PATCH 38/47] Codec door refuses only a proven family mismatch; source-generated formatters pass WrittenAsWalked required StandardResolver's formatter to equal one of the walk's own families, which refused every source-generated formatter. WrittenAgainstWalk asks one contrary resolver per walk kind and refuses only on a reference match. DcSeq is refused as a collection written as an object; FormattedRoles passes (registration owns the attribute). Co-Authored-By: Claude Opus 5.5 --- .../CultDocumentMessagePackSerialization.cs | 25 +++++++------- .../UnionArmGuardTests.cs | 33 ++++++++++++++++--- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 379d3e36..cd120a33 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -8,7 +8,6 @@ using System.Reflection; using GameCult.Caching; using MessagePack; using MessagePack.Formatters; -using MessagePack.ImmutableCollection; using MessagePack.Resolvers; namespace GameCult.Caching.MessagePack; @@ -97,10 +96,11 @@ public static class CultDocumentMessagePackSerialization return $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message})."; } - if (!WrittenAsWalked(type, generic, standard)) + if (WrittenAgainstWalk(type, generic, standard)) { - var owner = CultElementIds.WalksAsOwner(type); - return $"Cult document {documentType.Name}: MessagePack writes {type.Name} {(owner ? "as a collection" : "by its [Key] layout")}, but the element-id walk reads it {(owner ? "by its [Key] layout" : "as a collection")}, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other."; + return CultElementIds.WalksAsOwner(type) + ? $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other." + : $"Cult document {documentType.Name}: MessagePack writes {type.Name} as an object, but the element-id walk reads it as a collection, so element ids would be lost or invented. A [DataContract] type that is also enumerable is written by its members; mark it [MessagePackObject] with integer [Key]s, or drop the contract."; } var allowPrivateAsked = false; @@ -143,16 +143,15 @@ public static class CultDocumentMessagePackSerialization return null; } - // The walk's kind for a type on the id path, checked against the resolver MessagePack actually delegates to. The public - // singletons hand out the same cached formatter instances StandardResolver does, so a reference match names the family. A - // type the walk reads by key must be written by the object or union resolver, and one it reads as a collection by a - // collection resolver; a keyed class that is also an ICollection is written as a collection and its [Key] layout dropped. - private static bool WrittenAsWalked(Type type, MethodInfo generic, object? standard) + // The door refuses only a proven mismatch: StandardResolver's formatter being the very instance the contrary resolver's + // public singleton hands out (one is cached per type). An owner is contradicted by the generic resolver (a keyed + // ICollection or dictionary), a collection by the object resolver (a [DataContract] enumerable). Union types walk as + // owners and the immutable and builtin resolvers claim only framework types, so neither can contradict. Any other + // formatter, a source-generated one among them, passes: the walk owns which types carry ids and this door is its safety net. + private static bool WrittenAgainstWalk(Type type, MethodInfo generic, object? standard) { - var families = CultElementIds.WalksAsOwner(type) - ? new IFormatterResolver[] { DynamicUnionResolver.Instance, DynamicObjectResolver.Instance } - : new IFormatterResolver[] { DynamicGenericResolver.Instance, ImmutableCollectionResolver.Instance, BuiltinResolver.Instance }; - return families.Any(family => ReferenceEquals(Formatter(family, generic), standard)); + var contrary = CultElementIds.WalksAsOwner(type) ? (IFormatterResolver)DynamicGenericResolver.Instance : DynamicObjectResolver.Instance; + return standard != null && ReferenceEquals(Formatter(contrary, generic), standard); } private static object? Formatter(IFormatterResolver resolver, MethodInfo generic) diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 91c94d3c..760bf9ca 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.Serialization; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; @@ -380,7 +381,9 @@ namespace GameCult.Caching.Tests Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d")); } - // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. + // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. A + // [DataContract] list subclass is still written by the generic resolver; an object formatter the object resolver could also + // build (DcColl: a different instance) or fails to build (DcBad) is no mismatch. [Test] public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter() { @@ -395,6 +398,7 @@ namespace GameCult.Caching.Tests typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G), typeof(G), typeof(G), + typeof(G), typeof(G), }) { if (holder != typeof(G)) @@ -403,12 +407,22 @@ namespace GameCult.Caching.Tests } } - // The walk reads a List subclass as a collection; a type MessagePack writes by an attribute formatter instead is a mismatch. + // The attribute formatter is of no known family, so the door passes it as it passes a source-generated one; registration + // owns the attribute. [Test] - public void ACollectionMessagePackWritesByAnotherFormatterIsRefused() + public void AFormatterOfNoKnownFamilyPassesTheDoor() { - var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; - Assert.That(thrown.Message, Does.Contain($"writes {nameof(FormattedRoles)} by its [Key] layout, but the element-id walk reads it as a collection")); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G))); + var problems = CultElementIds.Inspect(typeof(G)).Problems; + Assert.That(problems, Has.Count.EqualTo(1)); + Assert.That(problems.Single(), Does.Contain("FormattedRoles").And.Contain("declares [MessagePackFormatter]")); + } + + [Test] + public void ACollectionMessagePackWritesAsAnObjectIsRefused() + { + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; + Assert.That(thrown.Message, Does.Contain("MessagePack writes DcSeq as an object, but the element-id walk reads it as a collection")); } // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter @@ -499,6 +513,15 @@ namespace GameCult.Caching.Tests public FormattedRoles Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new FormattedRoles(); } } + [DataContract] public sealed class DcSeq : IEnumerable + { + [DataMember(Order = 0)] public List Items = new(); + public IEnumerator GetEnumerator() => Items.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [DataContract] public sealed class DcColl : List { [DataMember(Order = 0)] public int N; } + [DataContract] public sealed class DcBad : List { [DataMember(Order = 0)] public int A; [DataMember(Order = 0)] public int B; } + public sealed class DkFormatter : IMessagePackFormatter { public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N); From d89dd26507ceb3ff7aaee8a6a812b78221bf9f42 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 20:52:25 +0200 Subject: [PATCH 39/47] GeometryDocumentTests: a generated formatter passes the element-id door Co-Authored-By: Claude Opus 5.5 --- .../GeometryDocumentTests.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs index b5f63775..a58bc29d 100644 --- a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs +++ b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs @@ -5,6 +5,7 @@ using GameCult.Caching.MessagePack; using GameCult.Mesh; using GameCult.Networking; using MessagePack; +using MessagePack.Resolvers; using NUnit.Framework; using R3; using System; @@ -193,6 +194,19 @@ namespace GameCult.Geometry.Tests .Should().Be("geometry:domain:02c9b5810977406b0c206f3a3494327a423abb9448192be1b9a1863cd0f2ed95"); } + // A source-generated formatter is of no known family and passes the element-id door (ruling c2a-source-generated-formatters-defaulted). + [Test] + public void AGeneratedFormatterPassesTheElementIdDoor() + { + StandardResolver.Instance.GetFormatter().Should() + .NotBeSameAs(DynamicObjectResolver.Instance.GetFormatter(), "the fixture must stay source-generated"); + var payload = File.ReadAllBytes(FixturePath("ragnarok-domain.msgpack")); + var domain = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultGeometryDomainDocument), payload); + var again = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped( + typeof(CultGeometryDomainDocument), CultDocumentMessagePackSerialization.SerializeUntyped(domain, typeof(CultGeometryDomainDocument))); + CultGeometryDomainDocument.CreateRecordKey(again).Value.Should().Be(CultGeometryDomainDocument.CreateRecordKey(domain).Value); + } + [Test] public void RustExportedChunkArtifact_DecodesWithStableCultCacheKey() { From 57fdf70c213a7189b7db9b86f1045b64c4f3edfc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 15 Oct 2026 20:52:37 +0200 Subject: [PATCH 40/47] Contracts: the codec door refuses only a proven family mismatch Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 4 +++- .../Contracts/cultcache-store-composition.md | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 966b2a1f..81c5eef3 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -447,7 +447,9 @@ elements retargets nothing. `[MessagePackObject]` type is read by key even when it is `IEnumerable`; MessagePack writes a keyed type that is also a collection (`ICollection`, `IList`, `IDictionary`) as a collection, so C# refuses such a type at the first encode or decode when it is on the - path to an object list. For an + path to an object list. A `[DataContract]` type the walk reads as a collection, which + MessagePack writes by its members, is refused the same way. A formatter MessagePack's + source generator emits is not refused. For an object list, C# refuses at registration a concrete element type that is not keyed and `[MessagePackFormatter]` on any type, collection type or member on the way to the list or on the id member, and refuses at the first encode or decode of the document a formatter diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index f2990a3a..60b5c4ca 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -303,11 +303,13 @@ exclusive open of the same lock file exclude each other is not established. type does not claim it (registration refuses a non-public `[Key]` member on the way to an object list, so the private-member formatter writes every id). A value list (`List`, a list of a non-keyed class) is not on the path, so a resolver may claim it. -- The codec also checks, for every type on that path, that `StandardResolver` writes it - the way the walk reads it. A type read by key must be written by the object or union - resolver; a collection must be written by a collection resolver (generic, immutable or - builtin). Otherwise the codec refuses the document on first encode or decode, naming the - type. It also refuses, typed, a type MessagePack cannot build a formatter for, and a +- The codec also checks, for every type on that path, that `StandardResolver` does not write it + against the walk. A type read by key may not be written by the generic collection resolver + (a `[MessagePackObject]` class that is also an `ICollection` or dictionary), and a type + read as a collection may not be written by the object resolver (a `[DataContract]` + enumerable); either is refused on first encode or decode, naming the type. Any other + formatter, such as one MessagePack's source generator emits, passes: the walk decides which + types carry ids. It also refuses, typed, a type MessagePack cannot build a formatter for, and a resolver that throws while asked about one. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. From 19f475ca9218e3a0af00c49988873a2ecbba630f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 00:54:37 +0200 Subject: [PATCH 41/47] Element ids part 2: one recogniser; shapes the walk does not own are whole values Delete Hidden, HiddenProblem, UnreadKeyedMembers/UnreadMembers, Settle's [MessagePackFormatter] refusals and the door's written-as-object branch. Classify gains Leaf for attribute-formatted, [DataContract] and string-keyed types; MembersOf honours the most derived AllowPrivate; the door refuses a null StandardResolver formatter with the throwing path's text. Co-Authored-By: Claude Opus 5.5 --- .../CultDocumentMessagePackSerialization.cs | 38 ++--- src/GameCult.Caching/CultElementIds.cs | 138 +++++++-------- .../ElementIdEngineTests.cs | 159 ++++++++++++++---- .../GameCult.Caching.Tests/ElementIdTests.cs | 51 ++++++ .../UnionArmGuardTests.cs | 58 +++++-- 5 files changed, 297 insertions(+), 147 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index cd120a33..ee5691f5 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -84,8 +84,8 @@ public static class CultDocumentMessagePackSerialization var generic = GetFormatterMethod.MakeGenericMethod(type); // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout - // plus non-public [Key] members, and registration refuses a non-public member on the way to an object list, so it - // claims nothing either. + // plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing + // either. object? standard; try { @@ -93,15 +93,14 @@ public static class CultDocumentMessagePackSerialization } catch (Exception exception) { - return $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message})."; + return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message); } - if (WrittenAgainstWalk(type, generic, standard)) - { - return CultElementIds.WalksAsOwner(type) - ? $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other." - : $"Cult document {documentType.Name}: MessagePack writes {type.Name} as an object, but the element-id walk reads it as a collection, so element ids would be lost or invented. A [DataContract] type that is also enumerable is written by its members; mark it [MessagePackObject] with integer [Key]s, or drop the contract."; - } + if (standard == null) + return CannotBuild(documentType, type, "StandardResolver has no formatter for it"); + + if (WrittenAsCollection(type, generic, standard)) + return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; var allowPrivateAsked = false; object? allowPrivate = null; @@ -136,23 +135,22 @@ public static class CultDocumentMessagePackSerialization if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate)) break; - return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout or keep it out of the document's lists."; + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; } } return null; } - // The door refuses only a proven mismatch: StandardResolver's formatter being the very instance the contrary resolver's - // public singleton hands out (one is cached per type). An owner is contradicted by the generic resolver (a keyed - // ICollection or dictionary), a collection by the object resolver (a [DataContract] enumerable). Union types walk as - // owners and the immutable and builtin resolvers claim only framework types, so neither can contradict. Any other - // formatter, a source-generated one among them, passes: the walk owns which types carry ids and this door is its safety net. - private static bool WrittenAgainstWalk(Type type, MethodInfo generic, object? standard) - { - var contrary = CultElementIds.WalksAsOwner(type) ? (IFormatterResolver)DynamicGenericResolver.Instance : DynamicObjectResolver.Instance; - return standard != null && ReferenceEquals(Formatter(contrary, generic), standard); - } + private static string CannotBuild(Type documentType, Type type, string reason) => + $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason})."; + + // The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key + // (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed + // ICollection or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework + // types, so neither can contradict. Any other formatter, a source-generated one among them, passes. + private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) => + CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard); private static object? Formatter(IFormatterResolver resolver, MethodInfo generic) { diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 4746b59c..99eb094a 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.Serialization; using System.Security.Cryptography; using System.Text; using MessagePack; @@ -109,10 +110,8 @@ namespace GameCult.Caching // can sit in a cycle, so the path is a fixed point over these edges and not a stack. private sealed class Graph { - // Hidden is true when the member that starts the entry is non-public: MessagePack writes it only under AllowPrivate, - // and the walk reads public members only. - public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through, bool Hidden)> Edges { get; } = new(); - public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through, bool Hidden)> Lists { get; } = new(); + public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through)> Lists { get; } = new(); // Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too. public List Enumerating { get; } = new(); @@ -140,17 +139,15 @@ namespace GameCult.Caching if (EnumeratesObjects(type)) graph.Enumerating.Add(type); foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, Array.Empty(), false, seen, survey, graph); - foreach (var (member, memberType) in UnreadKeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, Array.Empty(), true, seen, survey, graph); + VisitMemberType(documentType, type, member, memberType, Array.Empty(), seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) { - graph.Edges.Add((type, null, union.SubType, Array.Empty(), false)); + graph.Edges.Add((type, null, union.SubType, Array.Empty())); VisitObject(documentType, union.SubType, seen, survey, graph); } } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, bool hidden, HashSet seen, Survey survey, Graph graph) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; // A sequence or map whose element is itself (an unkeyed Node : IEnumerable) ends the type walk here; without @@ -161,17 +158,17 @@ namespace GameCult.Caching switch (kind) { case Kind.Leaf: + // A shape the walk does not own is one whole value, never refused (q3 amended). return; case Kind.Map: - VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), hidden, seen, survey, graph); + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); return; case Kind.Sequence when IsElement(inner): var concretes = ConcreteTypes(inner).ToList(); - graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray(), hidden)); + graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray())); foreach (var concrete in concretes) { - // A hidden list is refused as a member (Settle), not once per element type. - if (hidden || !survey.Judged.Add(concrete)) + if (!survey.Judged.Add(concrete)) continue; var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) @@ -181,10 +178,10 @@ namespace GameCult.Caching VisitObject(documentType, inner, seen, survey, graph); return; case Kind.Sequence: - VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), hidden, seen, survey, graph); + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); return; default: - graph.Edges.Add((owner, member, memberType, through, hidden)); + graph.Edges.Add((owner, member, memberType, through)); VisitObject(documentType, memberType, seen, survey, graph); return; } @@ -193,24 +190,21 @@ namespace GameCult.Caching private const string UnkeyedElementProblem = "is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists."; - // The id path: every type that reaches an object list, the list's element types and the concrete types under them. - // [MessagePackFormatter] on any of them, or on a member the walk descends through, would hand the bytes to a formatter - // that does not write the [Key] layout the id lives in. + // The id path: every type that reaches an object list through members MessagePack writes by key, the lists' element and + // concrete types, and the keyed types that enumerate objects. The codec verifies that MessagePack writes each of them the + // way Classify reads it; it never decides ownership. private static void Settle(Type documentType, Survey survey, Graph graph) { - var lists = graph.Lists.Where(list => !list.Hidden).ToArray(); + var lists = graph.Lists; var reaching = new HashSet(lists.Select(list => list.Owner)); - var members = new List(lists.Select(list => list.Member)); var through = new List(lists.SelectMany(list => list.Through)); for (var grew = true; grew;) { grew = false; - foreach (var (owner, member, child, via, hidden) in graph.Edges) + foreach (var (owner, _, child, via) in graph.Edges) { - if (hidden || !reaching.Contains(child)) + if (!reaching.Contains(child)) continue; - if (member != null && !members.Contains(member)) - members.Add(member); through.AddRange(via); grew |= reaching.Add(owner); } @@ -239,30 +233,9 @@ namespace GameCult.Caching foreach (var type in graph.Enumerating) Add(type); - foreach (var type in path) - { - survey.IdPath.Add(type); - if (type.IsDefined(typeof(MessagePackFormatterAttribute), true)) - survey.Problems.Add($"Cult document {documentType.Name}: {type.Name} is on the path to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute or keep this type out of the document's lists."); - } - - foreach (var member in members) - { - if (Declares(member, typeof(MessagePackFormatterAttribute))) - survey.Problems.Add($"Cult document {documentType.Name}: {member.DeclaringType!.Name}.{member.Name} leads to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute."); - } - - // The fixed point has settled which types reach a list through public members. A non-public [Key] member that starts - // a list, or leads to a type that reaches one, is refused: nothing reads or checks the ids it holds. - foreach (var list in graph.Lists.Where(list => list.Hidden)) - survey.Problems.Add(HiddenProblem(documentType, list.Owner, list.Member)); - foreach (var edge in graph.Edges.Where(edge => edge.Hidden && reaching.Contains(edge.Child))) - survey.Problems.Add(HiddenProblem(documentType, edge.Owner, edge.Member!)); + survey.IdPath.AddRange(path); } - private static string HiddenProblem(Type documentType, Type owner, MemberInfo member) => - $"Cult document {documentType.Name}: {owner.Name}.{member.Name} is a non-public [Key] member on the way to a list whose elements carry ids; MessagePack writes it only under AllowPrivate and the element-id walk reads public members, so its ids could never be checked. Make it public, or keep lists out of it."; - // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype // under it. Shapes are built from these, so registration judges exactly what minting will meet. private static IEnumerable ConcreteTypes(Type element) @@ -302,14 +275,14 @@ namespace GameCult.Caching if (KeyOf(id) is not { } idKey) return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted."; if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written) - return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], non-public, or a property whose getter is not public); the id would never reach disk."; + return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], or non-public on a type without [MessagePackObject(AllowPrivate = true)]); the id would never reach disk."; id = written; var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey); if (sharing.Member != null) return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own."; if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true }) return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it."; - if (id is PropertyInfo { SetMethod.IsPublic: false }) + if (id is PropertyInfo { SetMethod.IsPublic: false } && !AllowsPrivate(element)) return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine)."; var source = id.GetCustomAttribute(true)!.DerivedFrom; if (source != null) @@ -551,10 +524,20 @@ namespace GameCult.Caching return shape; } - // The members MessagePack's StandardResolver writes for an integer-keyed type: public instance fields, and public - // instance properties with a public getter, none of them [IgnoreMember]. TheWalkReadsTheMembersMessagePackWrites checks - // this set against MessagePack's own member discovery. + // The members MessagePack's StandardResolver writes for an integer-keyed type, none of them [IgnoreMember]. Without + // AllowPrivate: public instance fields, and public instance properties with a public getter. With it, read from the most + // derived [MessagePackObject] declaration (a derived type without it is written without it, probe NoApDerived): every + // instance field and indexer-free property up the base chain at any visibility, the most derived of a name. + // TheWalkReadsTheMembersMessagePackWrites checks this set against MessagePack's own member discovery. private static IEnumerable MembersOf(Type type) => + AllowsPrivate(type) + ? AllInstanceMembers(type).Where(member => !Declares(member, typeof(IgnoreMemberAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()) + : PublicMembersOf(type); + + private static bool AllowsPrivate(Type type) => type.GetCustomAttribute(true)?.AllowPrivate == true; + + private static IEnumerable PublicMembersOf(Type type) => type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast() .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public) .Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true })) @@ -573,36 +556,17 @@ namespace GameCult.Caching } } - // The members the walk reads, as MessagePack writes them, for the oracle test that checks both against MessagePack. - internal static IEnumerable WrittenKeyedMembers(Type type) => KeyedMembers(type).Select(entry => entry.Member); - - // The [Key] members MessagePack writes only under AllowPrivate, for the same test. - internal static IEnumerable UnreadMembers(Type type) => UnreadKeyedMembers(type).Select(entry => entry.Member); + // The [Key] members MessagePack writes, for the oracle test that checks them against MessagePack. + internal static IEnumerable WrittenKeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null); private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) => - MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member))); + WrittenKeyedMembers(type).Where(member => !Declares(member, typeof(MessagePackFormatterAttribute))) + .Select(member => (member, TypeOf(member))); // An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an // override of a base virtual member would not see its base's attribute. private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); - // [Key] members MessagePack writes only under AllowPrivate: non-public fields and properties, and public properties whose - // getter is not public, up the base chain. An [IgnoreMember] member is never written, so it is not one of them. - private static IEnumerable<(MemberInfo Member, Type Type)> UnreadKeyedMembers(Type type) - { - for (var level = type; level != null && level != typeof(object); level = level.BaseType) - { - const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; - foreach (var member in level.GetFields(flags).Where(field => !field.IsPublic).Cast() - .Concat(level.GetProperties(flags).Where(property => - property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: false }))) - { - if (KeyOf(member) != null && !Declares(member, typeof(IgnoreMemberAttribute))) - yield return (member, TypeOf(member)); - } - } - } - // True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it. internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner; @@ -623,18 +587,23 @@ namespace GameCult.Caching // [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable; the codec checks, for every type // on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not. // Inner is a map's value type, a sequence's element type, or the type itself for an owner. - // Leaf a value written whole that holds no element list: strings (an IEnumerable MessagePack writes whole), - // a struct, class or interface with no [MessagePackObject] or [Union] (primitives, enums, float2, DateTime, - // and object, which is untyped on disk). + // The owned shapes are a closed list (Owner, Sequence, Map); every other shape is Leaf: one whole value with no element + // ids, replaced whole by a variant (q3 amended 2026-10-16). When in doubt the answer is Leaf. + // Leaf strings (an IEnumerable MessagePack writes whole), a type written whole (IsWrittenWhole: attribute- + // formatted, [DataContract], string-keyed), a struct or a class or interface with no [MessagePackObject] or + // [Union] and no collection shape (primitives, enums, float2, DateTime, ImmutableArray, ArraySegment, Memory, + // KeyValuePair, tuples, Lazy, unkeyed classes), and object, which is untyped on disk. // Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>. - // Sequence an array or an IEnumerable. - // Owner a class, interface or struct MessagePack serializes by key. A keyed struct is walked as an owner; it is - // never an element (IsElement), since a boxed copy cannot carry a minted id. + // Sequence an array or an IEnumerable class or interface. + // Owner a class, interface or struct MessagePack serializes by integer key. A keyed struct is walked as an owner; + // it is never an element (IsElement), since a boxed copy cannot carry a minted id. private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type => { type = Nullable.GetUnderlyingType(type) ?? type; if (type == typeof(string)) return (Kind.Leaf, type); + if (IsWrittenWhole(type)) + return (Kind.Leaf, type); if (IsKeyed(type)) return (Kind.Owner, type); if (type.IsValueType) @@ -646,6 +615,15 @@ namespace GameCult.Caching return (Kind.Leaf, type); }); + // A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type, + // inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string + // keys (KeyAsPropertyName, or any [Key] with a string key). Such a type is Leaf, even when it is also enumerable. + private static bool IsWrittenWhole(Type type) => + type.IsDefined(typeof(MessagePackFormatterAttribute), true) || + (type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) || + type.GetCustomAttribute(true)?.KeyAsPropertyName == true || + AllInstanceMembers(type).Any(member => member.GetCustomAttribute(true)?.StringKey != null); + // A type MessagePack serializes by key: [MessagePackObject] or [Union]. private static bool IsKeyed(Type type) => type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true); diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 29fc497b..edb1c1c7 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -1,7 +1,9 @@ #nullable enable using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; +using System.Reflection; using System.Text.RegularExpressions; using GameCult.Caching; using MessagePack; @@ -302,16 +304,27 @@ namespace GameCult.Caching.Tests Assert.That(Problems(typeof(HValueList)), Is.Empty, "a list of a non-keyed type is a value list"); } + // [MessagePackFormatter] on an element type, an owner, a collection class or a member makes that value one whole value: it + // has no element ids and is off the id path. The attribute on the id member itself is the one refusal that stays. [Test] - public void AFormatterAttributeOnTheIdPathIsRefused() + public void AFormatterAttributeMakesAWholeValueExceptOnTheId() { - Assert.That(Problems(typeof(HFormattedElement)).Single(), Does.Contain("FormattedElement").And.Contain("[MessagePackFormatter]")); - Assert.That(Problems(typeof(HFormattedList)).Single(), Does.Contain("Items").And.Contain("[MessagePackFormatter]")); Assert.That(Problems(typeof(HFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); - Assert.That(Problems(typeof(HFormattedOwner)).Single(), Does.Contain("FormattedOwner").And.Contain("[MessagePackFormatter]")); - Assert.That(Problems(typeof(HFormattedMember)).Single(), Does.Contain("Inner").And.Contain("[MessagePackFormatter]")); + foreach (var (type, holder) in new (Type, object)[] + { + (typeof(HFormattedElement), new HFormattedElement { Items = { new FormattedElement() } }), + (typeof(HFormattedList), new HFormattedList { Items = { new EIded() } }), + (typeof(HFormattedOwner), new HFormattedOwner { Owner = { Items = { new EIded() } } }), + (typeof(HFormattedMember), new HFormattedMember { Inner = { Items = { new EIded() } } }), + (typeof(HFormattedDerived), new HFormattedDerived { Items = { new FormattedDerived() } }), + }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(0), type.Name); + } + Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field"); - Assert.That(Problems(typeof(HFormattedDerived)).Single(), Does.Contain("FormattedDerived").And.Contain("[MessagePackFormatter]"), "the attribute is inherited"); } // The collection types between a member and its object list are written by some formatter too, so they sit on the id path. @@ -331,18 +344,19 @@ namespace GameCult.Caching.Tests } [Test] - public void AFormatterOnACollectionClassOfAnObjectListIsRefused() + public void AFormatterOnACollectionClassOfAnObjectListIsAWholeValue() { - Assert.That(Problems(typeof(HRoles)).Single(), Does.Contain("Roles").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(HRoles)), Is.Empty); + Assert.That(Assign(new HRoles { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "its elements carry no ids and are never minted"); Assert.That(Problems(typeof(HFormattedValueCollection)), Is.Empty, "a collection class of a non-keyed element is a value list"); } // MemberInfo.IsDefined ignores inherit for properties: an override sees its base's attribute only through Declares. [Test] - public void AnInheritedFormatterOnAnOverriddenMemberIsRefused() + public void AnInheritedFormatterOnAnOverriddenMemberIsHonoured() { Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); - Assert.That(Problems(typeof(OwnerOverFormatted)).Single(), Does.Contain("Items").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(OwnerOverFormatted)), Is.Empty, "the override inherits its base's formatter, so the member is a whole value"); Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id"); var root = new HIdOverride { Items = { new IdOverride() } }; @@ -467,22 +481,28 @@ namespace GameCult.Caching.Tests Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1)); } - // MessagePack writes a non-public [Key] member only under AllowPrivate, and the element-id walk reads public members, so - // an object list a non-public member leads to could never be checked. It is refused; a non-public member that reaches no - // object list is nobody's business. + // MessagePack writes a non-public [Key] member only under AllowPrivate, so a non-public member of a type without it is nobody's + // id business; under AllowPrivate it is written, read and minted like any other member. [Test] - public void ANonPublicKeyedMemberOnTheWayToAnObjectListIsRefused() + public void ANonPublicMemberIsWalkedOnlyUnderAllowPrivate() { - foreach (var (type, member) in new[] + foreach (var type in new[] { typeof(HHiddenList), typeof(HHiddenOwner), typeof(HHiddenFromBase) }) { - (typeof(HHiddenList), "Stash"), (typeof(HHiddenOwner), "stash"), (typeof(HOptHidden), "stash"), (typeof(HHiddenFromBase), "stash"), - }) - Assert.That(Problems(type).Single(), Does.Contain(member).And.Contain("non-public"), type.Name); + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + Assert.That(Problems(typeof(HOptHidden)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HOptHidden)).IdPath, Is.EquivalentTo(new[] { typeof(HOptHidden), typeof(List), typeof(EIded) })); + var opt = new HOptHidden(); + opt.Peek.Add(new EIded()); + opt.Peek.Add(new EIded()); + Assert.That(Assign(opt, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(opt.Peek.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); Assert.That(Problems(typeof(HHiddenLeaf)), Is.Empty); Assert.That(CultElementIds.Inspect(typeof(HHiddenLeaf)).IdPath, Is.EquivalentTo(new[] { typeof(HHiddenLeaf), typeof(List), typeof(EIded) }), - "a hidden member adds nothing to the id path"); - Assert.That(CultElementIds.Inspect(typeof(HHiddenList)).IdPath, Is.Empty); + "a non-public member adds nothing to the id path"); } // MessagePack writes a [MessagePackObject] class that is only IEnumerable by its keys, so the walk reads it by key. A @@ -515,26 +535,24 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True); } - // The walk reads the members MessagePack writes: a public field or a property with a public getter, no [IgnoreMember]. - // Anything else that carries a [Key] is MessagePack's AllowPrivate business and, on the way to an object list, is refused. + // The walk reads the members MessagePack writes: without AllowPrivate a public field or a property with a public getter, with + // it every member up the base chain, the most derived [MessagePackObject] declaration deciding; no [IgnoreMember]. [Test] public void TheWalkReadsTheMembersMessagePackWrites() { Assert.That(Problems(typeof(PubIgn)), Is.Empty, "an ignored member is never written, so its list is nobody's business"); - Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not a hidden list"); - var held = Problems(typeof(PgHold)).Single(); - Assert.That(held, Does.Contain("Items").And.Contain("non-public")); + Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written"); + Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate"); - foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix) }) + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived) }) { - var written = MessagePackMembers(type, allowPrivate: false); - Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), Is.EquivalentTo(written), type.Name); - var readable = MessagePackMembers(type, allowPrivate: true); - Assert.That(CultElementIds.UnreadMembers(type).Select(member => member.Name), Is.EquivalentTo(readable.Except(written)), type.Name); + var allowPrivate = type.GetCustomAttribute(true)!.AllowPrivate; + Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), + Is.EquivalentTo(MessagePackMembers(type, allowPrivate)), type.Name); } - Assert.That(CultElementIds.UnreadMembers(typeof(MemberMix)).Select(member => member.Name), Is.EquivalentTo(new[] { "priv" })); - Assert.That(CultElementIds.UnreadMembers(typeof(PgHold)).Select(member => member.Name), Is.EquivalentTo(new[] { "Items" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" }), + "the derived declaration decides, so an AllowPrivate base adds nothing"); } // The names of the members MessagePack's own discovery reads for a type, by reflection into its internal type: the @@ -599,6 +617,69 @@ namespace GameCult.Caching.Tests } } + // Every shape the walk does not own is one whole value: no problem, off the id path, never minted into and never counted. + [Test] + public void AShapeTheWalkDoesNotOwnIsOneWholeValue() + { + const string Set = "aaaaaaaaaaaa"; + EIded E(string id) => new() { Id = id }; + var holders = new (Type Type, Func Make)[] + { + (typeof(Whole>), id => new Whole> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole?>), id => new Whole?> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole>), id => new Whole> { Value = new ArraySegment(new[] { E(id) }) }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>>), id => new Whole>> { Value = new("k", new List { E(id) }) }), + (typeof(Whole<(List, int)>), id => new Whole<(List, int)> { Value = (new List { E(id) }, 1) }), + (typeof(Whole, int>>), id => new Whole, int>> { Value = Tuple.Create(new List { E(id) }, 1) }), + (typeof(Whole>>), id => new Whole>> { Value = new Lazy>(() => new List { E(id) }) }), + (typeof(Whole), id => new Whole { Value = new NkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new SkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new DcHold { Items = { E(id) } } }), + (typeof(HRoles), id => new HRoles { Items = { E(id) } }), + (typeof(HFormattedList), id => new HFormattedList { Items = { E(id) } }), + (typeof(HHiddenList), id => new HHiddenList()), + }; + foreach (var (type, make) in holders) + { + Assert.That(Problems(type), Is.Empty, type.ToString()); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.ToString()); + Assert.That(CultElementIds.CanHoldIds(type), Is.False, type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: false), Is.EqualTo(0), type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: true), Is.EqualTo(0), type.ToString()); + Assert.That(CultElementIds.Holds(type, make(Set)), Is.False, type.ToString()); + } + + // Elements of a string-keyed type carry no id either, and an owned list of a bare element is still refused. + Assert.That(Problems(typeof(Whole>)), Is.Empty); + RefusesBare(typeof(Whole>)); + } + + // MessagePack writes an AllowPrivate type with every member up its base chain, and the most derived [MessagePackObject] + // declaration decides whether that is so. + [Test] + public void AnAllowPrivateTypeIsWalkedByTheMembersMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" })); + + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var held = new HOptHidden(); + held.Peek.Add(new EIded()); + Assert.That(Assign(held, "r", deterministic: false), Is.EqualTo(1)); + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(held, options), options); + Assert.That(back.Peek.Single().Id, Is.EqualTo(held.Peek.Single().Id), "the minted id of a private list survives a round trip"); + + Assert.That(Problems(typeof(HApPrivSet)), Is.Empty, "MessagePack reads an AllowPrivate type's private setter back"); + var privSet = new HApPrivSet { Items = { new EApPrivSet(), new EApPrivSet() } }; + Assert.That(Assign(privSet, "r", deterministic: false), Is.EqualTo(2)); + var round = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(privSet, options), options); + Assert.That(round.Items.Select(item => item.Id), Is.EqualTo(privSet.Items.Select(item => item.Id))); + Assert.That(round.Items.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public"), "without AllowPrivate it is still refused"); + } + private static string?[] Ids(object holder) => holder switch { HInit init => init.Items.Select(item => item.Id).ToArray(), @@ -885,12 +966,24 @@ namespace GameCult.Caching.Tests #pragma warning disable CS0169, CS0414, CS0649 [MessagePackObject] public class HHiddenList { [Key(0)] private List Stash { get; set; } = new(); } [MessagePackObject] public class HHiddenOwner { [Key(0)] private HHasIds stash = new(); } - [MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List stash = new(); } + [MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List stash = new(); [IgnoreMember] public List Peek => stash; } [MessagePackObject] public class HHiddenBase { [Key(0)] private List stash = new(); } [MessagePackObject] public class HHiddenFromBase : HHiddenBase { [Key(1)] public int N { get; set; } } [MessagePackObject] public class HHiddenLeaf { [Key(0)] public List Items { get; set; } = new(); [Key(1)] private int secret = 7; } + [MessagePackObject] public class ApBase { [Key(0)] private List baseStash = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApDerived : ApBase { [Key(1)] private List own = new(); [Key(2)] public int N { get; set; } [Key(3)] public List Pg { private get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApBase2 { [Key(0)] private List baseStash = new(); } + [MessagePackObject] public class NoApDerived : ApBase2 { [Key(1)] public int N { get; set; } } #pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public class Whole { [Key(0)] public T? Value { get; set; } } + public class NkHold { public List Items = new(); } + [MessagePackObject] public class SkHold { [Key("items")] public List Items = new(); } + [MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; } + [System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List Items = new(); } + [MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HApPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; } [MessagePackObject] public class HDerivedText { [Key(0)] public List Items { get; set; } = new(); } diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 70201d19..32a84677 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Reflection; @@ -356,6 +357,48 @@ namespace GameCult.Caching.Tests private static byte[] OldShapedReels(string label) => MessagePackSerializer.Serialize(new List { new() { Label = label, Marks = { new OldMark { Text = "x" }, new OldMark { Text = "y" } } } }); + // A holder the walk does not own (ImmutableArray here) is one whole value: its elements carry no element ids, so it never + // marks the store, and a variant replaces it whole (q3 amended 2026-10-16). + [Test] + public void AWholeListHolderNeverMarksTheStoreAndAVariantReplacesItWhole() + { + Assert.That(CultElementIds.CanHoldIds(typeof(IdWholeHolder)), Is.False); + var path = PathOf("whole-holder.cc"); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + cache.Commit(batch => batch.Upsert(typeof(IdWholeHolder), + new IdWholeHolder { Name = "base", Reels = ImmutableArray.Create(new IdReel { Label = "set", Id = HexA }, new IdReel { Label = "unset" }) }, + new CultRecordKey("base"))); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "ids inside a whole value mark nothing"); + + using (var fresh = OpenWith(path, false, typeof(IdWholeHolder))) + { + fresh.Commit(batch => batch.Upsert(typeof(IdWholeHolder), new IdWholeHolder { Name = "other" }, new CultRecordKey("other"))); + var back = fresh.Get(new CultRecordKey("base"))!; + Assert.That(back.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { HexA, "" }), "set ids are plain values, unset ids stay unset"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + var replacement = ImmutableArray.Create(new IdReel { Label = "v", Id = "eeeeeeeeeeee" }); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdWholeHolder.Reels), replacement) })); + Assert.That(cache.Get(new CultRecordKey("v"))!.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { "eeeeeeeeeeee" }), "replaced whole"); + Assert.That(cache.Get(new CultRecordKey("base"))!.Reels, Has.Length.EqualTo(2), "the base is untouched"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "a variant makes v2, and nothing in it holds element ids"); + using var again = OpenWith(path, false, typeof(IdWholeHolder)); + Assert.That(again.Get(new CultRecordKey("v"))!.Reels.Single().Id, Is.EqualTo("eeeeeeeeeeee")); + + var step = new CultVariantOverride(CultOverrideOp.Set, new[] { new CultPathStep(1, HexA) }, string.Empty, MessagePackSerializer.Serialize("x")); + var refused = Assert.Catch(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { step })))!; + Assert.That(refused.ToString(), Does.Contain("top-level Set overrides only")); + } + [Test] public void AStoreThatCanHoldElementIdsCarriesTheV3MarkerAndOneThatCannotStaysV1() { @@ -1287,6 +1330,14 @@ namespace GameCult.Caching.Tests [Key(1)] public int Depth { get; set; } } + [CultDocument("tests.element_id_whole_holder", "tests.element_id_whole_holder.v1")] + [MessagePackObject] + public sealed class IdWholeHolder + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public ImmutableArray Reels { get; set; } = ImmutableArray.Empty; + } + [CultDocument("tests.element_id_deck", "tests.element_id_deck.v1")] [MessagePackObject] public sealed class IdDeck diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 760bf9ca..48038c9c 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -381,9 +381,7 @@ namespace GameCult.Caching.Tests Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d")); } - // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. A - // [DataContract] list subclass is still written by the generic resolver; an object formatter the object resolver could also - // build (DcColl: a different instance) or fails to build (DcBad) is no mismatch. + // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. [Test] public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter() { @@ -398,7 +396,6 @@ namespace GameCult.Caching.Tests typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G>), typeof(G), typeof(G), typeof(G), - typeof(G), typeof(G), }) { if (holder != typeof(G)) @@ -407,22 +404,45 @@ namespace GameCult.Caching.Tests } } - // The attribute formatter is of no known family, so the door passes it as it passes a source-generated one; registration - // owns the attribute. + // One recogniser decides what the walk owns and the door only verifies it: the same types give no registration problem and a + // passing door. A type formatted by attribute or written by the object resolver is one whole value, off the id path. [Test] - public void AFormatterOfNoKnownFamilyPassesTheDoor() + public void OneRecogniserDecidesAndTheDoorAgrees() { - Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G))); - var problems = CultElementIds.Inspect(typeof(G)).Problems; - Assert.That(problems, Has.Count.EqualTo(1)); - Assert.That(problems.Single(), Does.Contain("FormattedRoles").And.Contain("declares [MessagePackFormatter]")); + foreach (var holder in new[] { typeof(G), typeof(G), typeof(G), typeof(G), typeof(G) }) + { + Assert.That(CultElementIds.Inspect(holder).Problems, Is.Empty, holder.ToString()); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + + foreach (var whole in new[] { typeof(G), typeof(G), typeof(G), typeof(G) }) + Assert.That(CultElementIds.Inspect(whole).IdPath, Is.Empty, whole.ToString()); + var ap = CultElementIds.Inspect(typeof(G)).IdPath; + Assert.That(ap, Does.Contain(typeof(ApTrueHold)).And.Contain(typeof(List))); + var seq = new G { Value = new DcSeq { Items = { new FbArm() } } }; + Assert.That(CultElementIds.Plan(seq.GetType(), seq, "r", deterministic: false).Count, Is.EqualTo(0)); } + // MessagePack throws for a type, or StandardResolver has no formatter for it (an interface or abstract class that is only + // IEnumerable): such a type cannot be persisted, so the door refuses it with the same typed text as a throwing build. [Test] - public void ACollectionMessagePackWritesAsAnObjectIsRefused() + public void ATypeMessagePackCannotBuildIsATypedRefusal() { - var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; - Assert.That(thrown.Message, Does.Contain("MessagePack writes DcSeq as an object, but the element-id walk reads it as a collection")); + foreach (var (holder, name) in new[] { (typeof(G), nameof(IArms)), (typeof(G), nameof(AbsArms)) }) + { + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString())!; + Assert.That(thrown.Message, Does.Contain(name).And.Contain("cannot build a formatter")); + } + } + + // A keyed type that enumerates lists is read by key and is on the id path, so the door checks it: MessagePack writes it as a + // collection, which loses the ids the walk would mint. + [Test] + public void AKeyedTypeThatEnumeratesListsIsCheckedByTheDoor() + { + Assert.That(CultElementIds.Inspect(typeof(G)).IdPath, Does.Contain(typeof(KLL))); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KLL)} as a collection")); } // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter @@ -519,6 +539,16 @@ namespace GameCult.Caching.Tests public IEnumerator GetEnumerator() => Items.GetEnumerator(); System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); } + public interface IArms : IEnumerable { } + public abstract class AbsArms : IEnumerable + { + public abstract IEnumerator GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public sealed class KLL : List> { [Key(0)] public int N; } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject(AllowPrivate = true)] public sealed class ApTrueHold { [Key(0)] private List items = new(); } +#pragma warning restore CS0169, CS0414, CS0649 [DataContract] public sealed class DcColl : List { [DataMember(Order = 0)] public int N; } [DataContract] public sealed class DcBad : List { [DataMember(Order = 0)] public int A; [DataMember(Order = 0)] public int B; } From 19963293ae422ed1e54b6e319fe84b8e752d6110 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 00:54:39 +0200 Subject: [PATCH 42/47] Contracts: whole-value shapes, closed list of owned shapes, door only verifies Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 61 ++++++++++--------- .../Contracts/cultcache-store-composition.md | 18 +++--- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 81c5eef3..7afc1abe 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -408,12 +408,18 @@ fifth slot: ## Element Ids -Every object that sits in a list, array or dictionary value (`IDictionary<,>` or -`IReadOnlyDictionary<,>`) of a registered document type, or of a keyed struct a document -holds, union subtypes included, carries an id: one string member with an integer `[Key]` -and `[CultElementId]`. The id identifies the element inside its list, so an override can -name it (a path step is `[slot, elementId]`) and a base edit that reorders or inserts -elements retargets nothing. +Every object that sits in a list, array or dictionary value of a shape the element-id walk +owns carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The +owned shapes are arrays, `IEnumerable` classes and interfaces, `IDictionary<,>` and +`IReadOnlyDictionary<,>` values, and `[MessagePackObject]` or `[Union]` types with integer +keys (a keyed struct included), read through the members MessagePack writes. Union subtypes +are included. The id identifies the element inside its list, so an override can name it (a +path step is `[slot, elementId]`) and a base edit that reorders or inserts elements +retargets nothing. + +Any other shape is one whole value. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked and never counted for `v3`. A +variant replaces it whole (operator, 2026-10-16). - A random id is 12 lowercase hex characters. A derived id (`[CultElementId(nameof(Source))]`, for content-addressed elements) is the @@ -432,28 +438,27 @@ elements retargets nothing. Python and Rust read `v3` and keep a `v3` header they found when they rewrite a store; they carry ids and do not decide when a document carries them. - An object list's elements are written by their `[Key]` layout, because the id is a - keyed member of the element. A list whose element type is not keyed - (`[MessagePackObject]` or `[Union]`) is a value list and is replaced whole; so is a list - of keyed structs, since a struct is a value and never an element, but an object list - inside a keyed struct is walked like any other. The walk follows declared member types: - a member declared `object` is untyped on disk, and a subclass's extra members under a - non-union base-typed member are not written, so neither is walked. The walk reads the - members MessagePack writes: an integer `[Key]`, no `[IgnoreMember]`, a public field or a - property whose getter is public. Any other `[Key]` member on the way to an object list - (non-public, or a property with a non-public getter) is refused at registration, because - MessagePack writes it only under `AllowPrivate` and the element-id walk reads public - members. An id member needs a public setter or `init`, so the id MessagePack writes is read - back; one with a non-public setter, or one MessagePack does not write, is refused. A - `[MessagePackObject]` type is read by key even when it is `IEnumerable`; MessagePack - writes a keyed type that is also a collection (`ICollection`, `IList`, `IDictionary`) - as a collection, so C# refuses such a type at the first encode or decode when it is on the - path to an object list. A `[DataContract]` type the walk reads as a collection, which - MessagePack writes by its members, is refused the same way. A formatter MessagePack's - source generator emits is not refused. For an - object list, C# refuses at registration a concrete element type that is not keyed and - `[MessagePackFormatter]` on any type, collection type or member on the way to the list or on the id - member, and refuses at the first encode or decode of the document a formatter - resolver that claims one of those types. + keyed member of the element. Value lists are replaced whole: a list of a non-keyed + class or struct, or of primitives. So is a list of keyed structs, since a struct is a + value and never an element, but an object list inside a keyed struct is walked like any + other. The walk follows declared member types: a member declared `object` is untyped on + disk, and a subclass's extra members under a non-union base-typed member are not written, + so neither is walked. Whole values are `ImmutableArray`, `ArraySegment`, + `Memory`, `KeyValuePair`, tuples, `Lazy`, a class or struct without + `[MessagePackObject]`, a `[DataContract]` type, a string-keyed type, a type or member + declaring `[MessagePackFormatter]`, and a non-public member of a type without + `AllowPrivate`. The members read are an integer `[Key]` with no `[IgnoreMember]`: a + public field or a property whose getter is public, or any visibility up the base chain + under the most derived `[MessagePackObject(AllowPrivate = true)]` declaration. An id member + needs a setter MessagePack calls (public or `init`; any under `AllowPrivate`), so the id + MessagePack writes is read back. Still refused: a bad id member (missing, duplicate, not + a string, no integer `[Key]`, `[MessagePackFormatter]` on it, one MessagePack does not + write or cannot assign, a key shared with another member, a bad derived source), a + concrete union arm that is not keyed, a keyed type that is also a collection + (`ICollection`, `IList`, `IDictionary`: MessagePack writes it as a collection, so C# + refuses it at the first encode or decode when it is on the path to an object list), a + type MessagePack has no formatter for, and a formatter resolver that claims a type on the + path. A formatter MessagePack's source generator emits is not refused. ## Canonical Semantic Schema Hash diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 60b5c4ca..15874f3d 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -300,17 +300,19 @@ exclusive open of the same lock file exclude each other is not established. Those are written by their `[Key]` layout so each element's id reaches disk, and the codec refuses the document on first encode or decode, naming the resolver and the type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a - type does not claim it (registration refuses a non-public `[Key]` member on the way to - an object list, so the private-member formatter writes every id). A value list (`List`, a list of a non-keyed class) is not on the + type does not claim it: the extra non-public members the `AllowPrivate` formatter writes + are whole values the walk does not own. A type or member declaring + `[MessagePackFormatter]` is a whole value off the path, the supported way to give a type its + own formatter. A value list (`List`, a list of a non-keyed class) is not on the path, so a resolver may claim it. - The codec also checks, for every type on that path, that `StandardResolver` does not write it against the walk. A type read by key may not be written by the generic collection resolver - (a `[MessagePackObject]` class that is also an `ICollection` or dictionary), and a type - read as a collection may not be written by the object resolver (a `[DataContract]` - enumerable); either is refused on first encode or decode, naming the type. Any other - formatter, such as one MessagePack's source generator emits, passes: the walk decides which - types carry ids. It also refuses, typed, a type MessagePack cannot build a formatter for, and a - resolver that throws while asked about one. + (a `[MessagePackObject]` class that is also an `ICollection` or dictionary); it is + refused on first encode or decode, naming the type. Any other formatter, such as one + MessagePack's source generator emits, passes: the walk decides which types carry ids and + the door only verifies. It also refuses, typed, a type MessagePack cannot build a formatter + for (it throws, or `StandardResolver` returns none), and a resolver that throws while asked + about one. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. From 157276cf6e6eecfa63625a082e06e52258dddd6d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 01:02:45 +0200 Subject: [PATCH 43/47] Test: the whole-holder variants override their names Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/ElementIdTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 32a84677..bee0a5d5 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -385,7 +385,7 @@ namespace GameCult.Caching.Tests { var replacement = ImmutableArray.Create(new IdReel { Label = "v", Id = "eeeeeeeeeeee" }); cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), - new[] { cache.Override(nameof(IdWholeHolder.Reels), replacement) })); + new[] { cache.Override(nameof(IdWholeHolder.Name), "v"), cache.Override(nameof(IdWholeHolder.Reels), replacement) })); Assert.That(cache.Get(new CultRecordKey("v"))!.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { "eeeeeeeeeeee" }), "replaced whole"); Assert.That(cache.Get(new CultRecordKey("base"))!.Reels, Has.Length.EqualTo(2), "the base is untouched"); } @@ -395,7 +395,7 @@ namespace GameCult.Caching.Tests Assert.That(again.Get(new CultRecordKey("v"))!.Reels.Single().Id, Is.EqualTo("eeeeeeeeeeee")); var step = new CultVariantOverride(CultOverrideOp.Set, new[] { new CultPathStep(1, HexA) }, string.Empty, MessagePackSerializer.Serialize("x")); - var refused = Assert.Catch(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { step })))!; + var refused = Assert.Catch(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { again.Override(nameof(IdWholeHolder.Name), "p"), step })))!; Assert.That(refused.ToString(), Does.Contain("top-level Set overrides only")); } From e5a4f60bc71c0cad1a1c803c35b7632e22d2fcef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 01:28:00 +0200 Subject: [PATCH 44/47] Tests: inherited attributes, AllowPrivate subclass, null-formatter reason Co-Authored-By: Claude Opus 5.5 --- .../GameCult.Caching.Tests/ElementIdEngineTests.cs | 14 +++++++++++++- tests/GameCult.Caching.Tests/UnionArmGuardTests.cs | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index edb1c1c7..4deee9c6 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -544,7 +544,7 @@ namespace GameCult.Caching.Tests Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written"); Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate"); - foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived) }) + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor) }) { var allowPrivate = type.GetCustomAttribute(true)!.AllowPrivate; Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), @@ -637,6 +637,8 @@ namespace GameCult.Caching.Tests (typeof(Whole), id => new Whole { Value = new NkHold { Items = { E(id) } } }), (typeof(Whole), id => new Whole { Value = new SkHold { Items = { E(id) } } }), (typeof(Whole), id => new Whole { Value = new DcHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpBase { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpDerived { Items = { E(id) } } }), (typeof(HRoles), id => new HRoles { Items = { E(id) } }), (typeof(HFormattedList), id => new HFormattedList { Items = { E(id) } }), (typeof(HHiddenList), id => new HHiddenList()), @@ -651,6 +653,9 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Holds(type, make(Set)), Is.False, type.ToString()); } + // A [DataContract] subclass of a [MessagePackObject] type is still read by key: the attribute it inherits keeps it an owner. + Assert.That(CultElementIds.Inspect(typeof(Whole)).IdPath, Does.Contain(typeof(List))); + // Elements of a string-keyed type carry no id either, and an owned list of a bare element is still refused. Assert.That(Problems(typeof(Whole>)), Is.Empty); RefusesBare(typeof(Whole>)); @@ -663,6 +668,8 @@ namespace GameCult.Caching.Tests { Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" })); Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApInheritor)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" }), + "AllowPrivate is inherited by a subclass that declares no attribute of its own"); var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; var held = new HOptHidden(); @@ -979,6 +986,11 @@ namespace GameCult.Caching.Tests [MessagePackObject] public class Whole { [Key(0)] public T? Value { get; set; } } public class NkHold { public List Items = new(); } [MessagePackObject] public class SkHold { [Key("items")] public List Items = new(); } + [MessagePackObject(keyAsPropertyName: true)] public class KpBase { public List Items = new(); } + public class KpDerived : KpBase { } + [MessagePackObject] public class MpBase { [Key(0)] public List Items = new(); } + [System.Runtime.Serialization.DataContract] public class DcOverMp : MpBase { } + public class ApInheritor : ApDerived { } [MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; } [System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List Items = new(); } [MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index 48038c9c..85d17832 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -432,6 +432,7 @@ namespace GameCult.Caching.Tests { var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString())!; Assert.That(thrown.Message, Does.Contain(name).And.Contain("cannot build a formatter")); + Assert.That(thrown.Message, Does.Contain("StandardResolver has no formatter for it"), "a null formatter says so, without a thrown message"); } } From 7e69daa85e4dcae2358418916e0437ceb502cc5f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 01:38:13 +0200 Subject: [PATCH 45/47] Test: a KeyAsPropertyName type with an integer key is still one whole value Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/ElementIdEngineTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 4deee9c6..6fc04aad 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -986,7 +986,7 @@ namespace GameCult.Caching.Tests [MessagePackObject] public class Whole { [Key(0)] public T? Value { get; set; } } public class NkHold { public List Items = new(); } [MessagePackObject] public class SkHold { [Key("items")] public List Items = new(); } - [MessagePackObject(keyAsPropertyName: true)] public class KpBase { public List Items = new(); } + [MessagePackObject(keyAsPropertyName: true)] public class KpBase { [Key(0)] public List Items = new(); } public class KpDerived : KpBase { } [MessagePackObject] public class MpBase { [Key(0)] public List Items = new(); } [System.Runtime.Serialization.DataContract] public class DcOverMp : MpBase { } From 0c797ec3f714942e3d6eb233c919336d7c9e1a7a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 02:16:32 +0200 Subject: [PATCH 46/47] Element ids: string keys read from the members MessagePack writes; AllowPrivate, inherited-formatter and unwritten-string-key pins; README states the whole-value exception Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultElementIds.cs | 4 +- src/GameCult.Caching/README.md | 9 ++++ .../ElementIdEngineTests.cs | 41 ++++++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 99eb094a..89ff651b 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -617,12 +617,12 @@ namespace GameCult.Caching // A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type, // inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string - // keys (KeyAsPropertyName, or any [Key] with a string key). Such a type is Leaf, even when it is also enumerable. + // keys (KeyAsPropertyName, or a [Key] with a string key on a member MessagePack writes: a private or [IgnoreMember] one is not written). Such a type is Leaf, even when it is also enumerable. private static bool IsWrittenWhole(Type type) => type.IsDefined(typeof(MessagePackFormatterAttribute), true) || (type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) || type.GetCustomAttribute(true)?.KeyAsPropertyName == true || - AllInstanceMembers(type).Any(member => member.GetCustomAttribute(true)?.StringKey != null); + MembersOf(type).Any(member => member.GetCustomAttribute(true)?.StringKey != null); // A type MessagePack serializes by key: [MessagePackObject] or [Union]. private static bool IsKeyed(Type type) => diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index 6e47742f..fc5cd308 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -195,6 +195,15 @@ A document type that holds a list of objects gives every object type in that lis `[Key]`. The id is what lets a variant override one list element instead of the whole list. Lists of strings, numbers and other non-object values need none. +The rule covers the shapes the id walk owns: arrays, `IEnumerable` classes and +interfaces, dictionary values, and integer-keyed `[MessagePackObject]` and `[Union]` +types. Any other holder is one whole value: `ImmutableArray`, `ArraySegment`, +`Memory`, `KeyValuePair`, tuples, `Lazy`, a class without `[MessagePackObject]`, +a `[DataContract]` or string-keyed type, and a type or member declaring +`[MessagePackFormatter]`. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked, and a variant can only +replace the holder whole. + ```csharp [MessagePackObject] public sealed class Line diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 6fc04aad..3e0283fa 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -357,6 +357,8 @@ namespace GameCult.Caching.Tests { Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); Assert.That(Problems(typeof(OwnerOverFormatted)), Is.Empty, "the override inherits its base's formatter, so the member is a whole value"); + Assert.That(CultElementIds.Inspect(typeof(OwnerOverFormatted)).IdPath, Is.Empty, "a walked override would put its list on the id path"); + Assert.That(Assign(new OwnerOverFormatted { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "and mint into a list its formatter may drop ids from"); Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id"); var root = new HIdOverride { Items = { new IdOverride() } }; @@ -544,7 +546,7 @@ namespace GameCult.Caching.Tests Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written"); Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate"); - foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor) }) + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor), typeof(ApIgnAp), typeof(PSkH), typeof(ISkH) }) { var allowPrivate = type.GetCustomAttribute(true)!.AllowPrivate; Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), @@ -661,6 +663,33 @@ namespace GameCult.Caching.Tests RefusesBare(typeof(Whole>)); } + // A string key counts only on a member MessagePack writes: a private or [IgnoreMember] one leaves the type integer-keyed and owned. + [Test] + public void AStringKeyOnAMemberMessagePackDoesNotWriteLeavesTheTypeOwned() + { + foreach (var holder in new object[] { new PSkH { Items = { new EIded(), new EIded() } }, new ISkH { Items = { new EIded(), new EIded() } } }) + { + Assert.That(Problems(holder.GetType()), Is.Empty, holder.GetType().Name); + Assert.That(CultElementIds.Inspect(holder.GetType()).IdPath, Does.Contain(holder.GetType()), holder.GetType().Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + Assert.That(CultElementIds.Holds(holder.GetType(), holder), Is.True, holder.GetType().Name); + } + + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]")); + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]"), "an element whose only string key is ignored is as bare as its twin"); + } + + // The AllowPrivate edges: an [IgnoreMember] member is not written even under it, and only the element's own most derived + // declaration lets a private setter be read back. + [Test] + public void TheAllowPrivateEdgesFollowWhatMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApIgnAp)), Is.Empty, "an ignored member is not written under AllowPrivate"); + Assert.That(Problems(typeof(HApIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HNoApPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id"), + "the derived plain [MessagePackObject] decides: its private setter is not read back, whatever its base declares"); + } + // MessagePack writes an AllowPrivate type with every member up its base chain, and the most derived [MessagePackObject] // declaration decides whether that is so. [Test] @@ -993,6 +1022,16 @@ namespace GameCult.Caching.Tests public class ApInheritor : ApDerived { } [MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; } [System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List Items = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIgnAp { [Key(0)] [IgnoreMember] private List l = new(); } + [MessagePackObject] public class PSkH { [Key(0)] public List Items = new(); [Key("x")] private int x = 1; } + [MessagePackObject] public class ISkH { [Key(0)] public List Items = new(); [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class NoIdEl { [Key(0)] public int N { get; set; } [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class NoIdPlain { [Key(0)] public int N { get; set; } } + [MessagePackObject(AllowPrivate = true)] public class EApIgnId { [Key(0)] public int N { get; set; } [Key(1)] [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HApIgnId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIdBase { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class ENoApPrivSet : ApIdBase { [Key(1)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HNoApPrivSet { [Key(0)] public List Items { get; set; } = new(); } [MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } [MessagePackObject] public class HApPrivSet { [Key(0)] public List Items { get; set; } = new(); } From 64ec14f9891d5a875ea421a13ce47923243f5470 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 02:27:52 +0200 Subject: [PATCH 47/47] Element ids: pin the inherited string key that makes a type one whole value Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/ElementIdEngineTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 3e0283fa..2bc1d9df 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -675,6 +675,8 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Holds(holder.GetType(), holder), Is.True, holder.GetType().Name); } + Assert.That(CultElementIds.WalksAsOwner(typeof(SkOver)), Is.False, "a string key inherited by an override still makes the type one whole value"); + Assert.That(CultElementIds.WalksAsOwner(typeof(PSkH)), Is.True); Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]")); Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]"), "an element whose only string key is ignored is as bare as its twin"); } @@ -1025,6 +1027,8 @@ namespace GameCult.Caching.Tests [MessagePackObject(AllowPrivate = true)] public class ApIgnAp { [Key(0)] [IgnoreMember] private List l = new(); } [MessagePackObject] public class PSkH { [Key(0)] public List Items = new(); [Key("x")] private int x = 1; } [MessagePackObject] public class ISkH { [Key(0)] public List Items = new(); [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class SkOverBase { [Key("items")] public virtual List Items { get; set; } = new(); } + [MessagePackObject] public class SkOver : SkOverBase { public override List Items { get; set; } = new(); } [MessagePackObject] public class NoIdEl { [Key(0)] public int N { get; set; } [IgnoreMember] [Key("x")] public int X { get; set; } } [MessagePackObject] public class NoIdPlain { [Key(0)] public int N { get; set; } } [MessagePackObject(AllowPrivate = true)] public class EApIgnId { [Key(0)] public int N { get; set; } [Key(1)] [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; }