diff --git a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs index 4fa55076..8d0c7c96 100644 --- a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs +++ b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs @@ -23,10 +23,10 @@ namespace GameCult.Caching.MessagePack // options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type // deserializes as a registered document, a member type as itself. public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec( - (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsForDocument(document)), (type, document, bytes) => type == document ? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry) - : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsForDocument(document)), CultDocumentMessagePackSerialization.OverlaySlots); // The inspection model over that codec. diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index a10a0d62..ee5691f5 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -38,11 +38,23 @@ public static class CultDocumentMessagePackSerialization private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; - public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty()); + private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty()); - private static readonly ConcurrentDictionary AssemblyOptions = new(); + public static readonly MessagePackSerializerOptions Options = Plain.Options; - public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) + // Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of + // StandardResolver, the ones that can claim a type before its [Key] layout does. + private static readonly ConcurrentDictionary AssemblyOptions = new(); + + // Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers + // are fixed for the process, so this is a cache of a pure answer. + private static readonly ConcurrentDictionary DocumentProblems = new(); + + private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!; + + public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options; + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly) { if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly)); return AssemblyOptions.GetOrAdd(documentAssembly, static assembly => @@ -50,19 +62,118 @@ public static class CultDocumentMessagePackSerialization var resolvers = assembly.GetCustomAttributes() .Select(attribute => CreateResolver(attribute.ResolverType)) .ToArray(); - return resolvers.Length == 0 ? Options : Compose(resolvers); + return resolvers.Length == 0 ? Plain : Compose(resolvers); }); } - private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers) + // The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key] + // layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them + // without the id: refused here, once per document type, on every encode and decode path. + internal static MessagePackSerializerOptions OptionsForDocument(Type documentType) + { + if (documentType == null) throw new ArgumentNullException(nameof(documentType)); + var (options, ahead) = ComposedFor(documentType.Assembly); + var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead)); + return problem == null ? options : throw new InvalidOperationException(problem); + } + + private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead) + { + foreach (var type in CultElementIds.Inspect(documentType).IdPath) + { + var generic = GetFormatterMethod.MakeGenericMethod(type); + // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the + // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout + // plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing + // either. + object? standard; + try + { + standard = generic.Invoke(StandardResolver.Instance, null); + } + catch (Exception exception) + { + return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message); + } + + if (standard == null) + return CannotBuild(documentType, type, "StandardResolver has no formatter for it"); + + if (WrittenAsCollection(type, generic, standard)) + return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; + + var allowPrivateAsked = false; + object? allowPrivate = null; + foreach (var resolver in ahead) + { + object? formatter; + try + { + formatter = generic.Invoke(resolver, null); + } + catch (Exception exception) + { + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type."; + } + + if (formatter == null) + continue; + if (ReferenceEquals(formatter, standard)) + break; + if (!allowPrivateAsked) + { + allowPrivateAsked = true; + try + { + allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null); + } + catch (Exception) + { + allowPrivate = null; + } + } + + if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate)) + break; + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; + } + } + + return null; + } + + private static string CannotBuild(Type documentType, Type type, string reason) => + $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason})."; + + // The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key + // (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed + // ICollection or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework + // types, so neither can contradict. Any other formatter, a source-generated one among them, passes. + private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) => + CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard); + + private static object? Formatter(IFormatterResolver resolver, MethodInfo generic) + { + try + { + return generic.Invoke(resolver, null); + } + catch (Exception) + { + return null; + } + } + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers) { // The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer // resolver still encodes an arm while the key check runs. - var inner = CompositeResolver.Create( - consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).Append(StandardResolver.Instance).ToArray()); - return MessagePackSerializerOptions.Standard + var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray(); + var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray()); + var options = MessagePackSerializerOptions.Standard .WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner)) .WithSecurity(CultMessagePackSecurity.Instance); + return (options, ahead); } private static IFormatterResolver CreateResolver(Type resolverType) @@ -84,7 +195,7 @@ public static class CultDocumentMessagePackSerialization { if (registry == null) throw new ArgumentNullException(nameof(registry)); if (value != null) registry.GetRequired(type); - var payload = MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly)); + var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type)); CultPayloadDepth.Check(payload, type, PayloadDirection.Write); return payload; } @@ -99,7 +210,7 @@ public static class CultDocumentMessagePackSerialization if (registry == null) throw new ArgumentNullException(nameof(registry)); registry.GetRequired(type); CultPayloadDepth.Check(payload, type, PayloadDirection.Read); - return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly)) + return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type)) ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index eeec3349..76706417 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -363,6 +363,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentManifest.FormatVersion, holdsIds.Values.Select(holds => (bool?)holds).ToArray(), copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)), + copiedHoldIds: false, // only C# writes a directory store, and it marks by content directoryStore: true, holdsVariants: false); WriteManifest(targetCatalog, catalogBytes, currentIndex.Values diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index e1d5d0c0..7afc1abe 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -336,8 +336,11 @@ an element id, variants or not). A single-file store declares `v3` when any reco holds carries an element id: a non-empty `[CultElementId]` value in a record's document, or in a variant's override values. A store that cannot read a variant's override values (no codec attached) counts the variant as holding one. An element id is an ordinary keyed -string in the payload, so no reader can tell from a record's bytes whether it holds one; -a writer knows it only for the records it encodes. A write that copies any record forward +string in the payload, so only a reader with the record's type can tell; a C# write +decides for the records it encodes and, while the header it found is unmarked, for every +plain record it copies forward that its registry can read, decoding the bytes the file +holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload, +decide nothing. A write that copies any record forward therefore keeps a `v3` header it found. Only a write that encodes every record the store will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds an id. A directory store's manifest says @@ -405,11 +408,18 @@ fifth slot: ## Element Ids -Every object that sits in a list, array or dictionary value of a registered document -type, union subtypes included, carries an id: one string member with an integer `[Key]` -and `[CultElementId]`. The id identifies the element inside its list, so an override can -name it (a path step is `[slot, elementId]`) and a base edit that reorders or inserts -elements retargets nothing. +Every object that sits in a list, array or dictionary value of a shape the element-id walk +owns carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The +owned shapes are arrays, `IEnumerable` classes and interfaces, `IDictionary<,>` and +`IReadOnlyDictionary<,>` values, and `[MessagePackObject]` or `[Union]` types with integer +keys (a keyed struct included), read through the members MessagePack writes. Union subtypes +are included. The id identifies the element inside its list, so an override can name it (a +path step is `[slot, elementId]`) and a base edit that reorders or inserts elements +retargets nothing. + +Any other shape is one whole value. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked and never counted for `v3`. A +variant replaces it whole (operator, 2026-10-16). - A random id is 12 lowercase hex characters. A derived id (`[CultElementId(nameof(Source))]`, for content-addressed elements) is the @@ -423,8 +433,32 @@ elements retargets nothing. the record's next write, or `CultCache.MintElementIds()`. - A store that holds an id declares the `v3` header (above), so a reader older than ids refuses it instead of skipping the id slot and rewriting the elements without ids. - TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they - rewrite a store; they do not decide when a document carries ids. + A C# write also marks the store when a record it copies forward holds an id, judged + from the file's bytes at the write (above). TypeScript, + Python and Rust read `v3` and keep a `v3` header they found when they rewrite a + store; they carry ids and do not decide when a document carries them. +- An object list's elements are written by their `[Key]` layout, because the id is a + keyed member of the element. Value lists are replaced whole: a list of a non-keyed + class or struct, or of primitives. So is a list of keyed structs, since a struct is a + value and never an element, but an object list inside a keyed struct is walked like any + other. The walk follows declared member types: a member declared `object` is untyped on + disk, and a subclass's extra members under a non-union base-typed member are not written, + so neither is walked. Whole values are `ImmutableArray`, `ArraySegment`, + `Memory`, `KeyValuePair`, tuples, `Lazy`, a class or struct without + `[MessagePackObject]`, a `[DataContract]` type, a string-keyed type, a type or member + declaring `[MessagePackFormatter]`, and a non-public member of a type without + `AllowPrivate`. The members read are an integer `[Key]` with no `[IgnoreMember]`: a + public field or a property whose getter is public, or any visibility up the base chain + under the most derived `[MessagePackObject(AllowPrivate = true)]` declaration. An id member + needs a setter MessagePack calls (public or `init`; any under `AllowPrivate`), so the id + MessagePack writes is read back. Still refused: a bad id member (missing, duplicate, not + a string, no integer `[Key]`, `[MessagePackFormatter]` on it, one MessagePack does not + write or cannot assign, a key shared with another member, a bad derived source), a + concrete union arm that is not keyed, a keyed type that is also a collection + (`ICollection`, `IList`, `IDictionary`: MessagePack writes it as a collection, so C# + refuses it at the first encode or decode when it is on the path to an object list), a + type MessagePack has no formatter for, and a formatter resolver that claims a type on the + path. A formatter MessagePack's source generator emits is not refused. ## Canonical Semantic Schema Hash diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index b03627ca..15874f3d 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -294,6 +294,25 @@ exclusive open of the same lock file exclude each other is not established. for `X` that `Doc` needs is declared in `Doc`'s assembly. - An assembly that declares nothing, and the store envelope, use the base options. +- A resolver may not claim a type a document's element lists pass through: the list's + element types, the types and members on the way to them, and the collection types + the walk passes (`List`, `T[]`, a collection class, a dictionary on the way). + Those are written by their `[Key]` layout so each element's id reaches disk, and the + codec refuses the document on first encode or decode, naming the resolver and the + type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a + type does not claim it: the extra non-public members the `AllowPrivate` formatter writes + are whole values the walk does not own. A type or member declaring + `[MessagePackFormatter]` is a whole value off the path, the supported way to give a type its + own formatter. A value list (`List`, a list of a non-keyed class) is not on the + path, so a resolver may claim it. +- The codec also checks, for every type on that path, that `StandardResolver` does not write it + against the walk. A type read by key may not be written by the generic collection resolver + (a `[MessagePackObject]` class that is also an `ICollection` or dictionary); it is + refused on first encode or decode, naming the type. Any other formatter, such as one + MessagePack's source generator emits, passes: the walk decides which types carry ids and + the door only verifies. It also refuses, typed, a type MessagePack cannot build a formatter + for (it throws, or `StandardResolver` returns none), and a resolver that throws while asked + about one. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 7f164e4c..c037626d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -545,11 +545,14 @@ namespace GameCult.Caching get { if (Variant == null) - return CultElementIds.Holds(_document); + return CultElementIds.Holds(Descriptor.DocumentType, _document); if (_codec == null) return true; - return Variant.Overrides.Any(entry => CultElementIds.Holds(_codec.Deserialize( - Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType, Descriptor.DocumentType, entry.Value))); + return Variant.Overrides.Any(entry => + { + var memberType = Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType; + return CultElementIds.Holds(memberType, _codec.Deserialize(memberType, Descriptor.DocumentType, entry.Value)); + }); } } @@ -2734,7 +2737,7 @@ namespace GameCult.Caching // again, and lands in memory only (variants mint inside Resolve, into their override values). var idPlans = admitted .Where(stored => stored.Variant == null) - .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Document, stored.Key.Value, deterministic: source != null))) + .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Descriptor.DocumentType, stored.Document, stored.Key.Value, deterministic: source != null))) .ToArray(); VariantPlan plan; CultCommitOutcome outcome; @@ -3176,7 +3179,7 @@ namespace GameCult.Caching // The override's value is minted like a plain record's members: ids the store never held are filled here, in the // decoded copy, and the delta this hold lands carries them. var bytes = entry.Value; - var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); + var idPlan = CultElementIds.Plan(member.MemberType, value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); if (idPlan.Count > 0) { idPlan.Apply(); @@ -3491,6 +3494,9 @@ namespace GameCult.Caching protected CultDocumentRegistry Registry => _registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry."); + // Null for a store no registry was attached to (CultMesh's raw single-file write). + private protected CultDocumentRegistry? AttachedRegistry => _registry; + protected ConcurrentDictionary Entries { get; } = new(StringComparer.Ordinal); @@ -3542,13 +3548,16 @@ namespace GameCult.Caching return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1; } - // The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an - // element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any + // The header a write leaves, for either store kind: HeaderFor decides. An element id is an ordinary keyed string, so only a + // reader with the record's type can tell from its bytes whether it holds one; the store header is how a reader without the types + // learns it. A write that copies any // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An - // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. - protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool directoryStore, bool holdsVariants) => + // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes + // under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may + // have written ids under an unmarked header. Nothing a load or a decode records can unmark a store. + protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) => HeaderFor( - holdsIds: written.Any(value => value == true), + holdsIds: copiedHoldIds || written.Any(value => value == true), durableHeader, wholeStore: !copiesAny && written.All(value => value != null), directoryStore, @@ -3968,6 +3977,9 @@ namespace GameCult.Caching // Test seam: how many record encodings this store has hashed, to prove what a write hashes. internal int RecordsHashed; + // Test seam: how many copied records a write decoded to ask whether they hold an element id. + internal int CopiedRecordsDecoded; + private string VersionOfBytes(byte[] bytes) { RecordsHashed++; @@ -4206,7 +4218,8 @@ namespace GameCult.Caching // write that then has nothing to write but keys it checked writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions - // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is + // only: a write hashes other records' bytes, never decodes them. That holds for the variant gate; the one decode of a copied + // record is the element-id check (CopiedHoldIds), which runs only on an unmarked store and can only mark it. Otherwise the whole write is refused and what it refused is // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of // the bytes written. @@ -4250,6 +4263,7 @@ namespace GameCult.Caching durable?.Snapshot.FormatVersion, upserts.Select(staged => staged.HoldsIds).ToArray(), copiesAny: copied.Length > 0, + copiedHoldIds: CopiedHoldIds(durable, copied, upserts), directoryStore: false, holdsVariants: records.Any(record => record.Variant != null)); @@ -4263,6 +4277,43 @@ namespace GameCult.Caching return versions; } + // Whether a record this write copies forward holds an element id, decided from the bytes the file holds under the store's + // lock, so every C# writer with a type for the record decides the same from the same bytes. Positive-only: it runs only + // while the header on disk is unmarked and can only mark the store, so a narrower type that decodes no ids changes nothing. + // It never refuses a write: a record that does not resolve or decode is copied as its bytes and decides nothing. Gates, in + // order, so the decode runs only where it can change the answer. + private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection upserts) + { + if (durable == null || + string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) || + upserts.Any(staged => staged.HoldsIds == true) || + AttachedRegistry is not { } registry) + return false; + var catalog = durable.Snapshot.SchemaCatalog; + foreach (var record in copied) + { + if (record.Variant != null) + continue; + try + { + if (registry.Claimant(record.SchemaId, catalog, out _) == null) + continue; + var type = registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog).Descriptor.DocumentType; + if (!CultElementIds.CanHoldIds(type)) + continue; + CopiedRecordsDecoded++; + if (CultElementIds.Holds(type, DeserializePayload(type, record.Payload))) + return true; + } + catch (Exception e) when (e is not OperationCanceledException) + { + // Copied as its bytes: no type, or no decode, is no fact. + } + } + + return false; + } + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { var records = durable?.Snapshot.Records ?? Array.Empty(); diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 8e10198b..89ff651b 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.Serialization; using System.Security.Cryptography; using System.Text; using MessagePack; @@ -85,7 +86,7 @@ namespace GameCult.Caching public Action? SetId; public Func? Derive; public string? DerivedFrom; - public (int Slot, Func Get)[] Walk = Array.Empty<(int, Func)>(); + public (int Slot, Type Type, Func Get)[] Walk = Array.Empty<(int, Type, Func)>(); } private static readonly ConcurrentDictionary Shapes = new(); @@ -95,57 +96,144 @@ namespace GameCult.Caching internal sealed class Survey { public List Problems { get; } = new(); + + // Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's + // element types and every concrete type under them. A serialization codec must write all of these by their [Key] + // layout, so it checks them against the formatters it composed. + public List IdPath { get; } = new(); + + // The concrete element types already judged in this document, so each is refused at most once however many lists hold it. + internal HashSet Judged { get; } = new(); } + // What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type + // can sit in a cycle, so the path is a fixed point over these edges and not a stack. + private sealed class Graph + { + public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through)> Lists { get; } = new(); + + // Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too. + public List Enumerating { get; } = new(); + } + + private static readonly ConcurrentDictionary IdCapable = new(); + + // A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask. + internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0); + // Every reason this document type cannot be registered under the id rule, one per offending concrete element type. internal static Survey Inspect(Type documentType) { var survey = new Survey(); - VisitObject(documentType, documentType, new HashSet(), survey); + var graph = new Graph(); + VisitObject(documentType, documentType, new HashSet(), survey, graph); + Settle(documentType, survey, graph); return survey; } - private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey) + private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey, Graph graph) { if (!seen.Add(type)) return; + if (EnumeratesObjects(type)) + graph.Enumerating.Add(type); foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, seen, survey); + VisitMemberType(documentType, type, member, memberType, Array.Empty(), seen, survey, graph); foreach (var union in type.GetCustomAttributes(true)) - VisitObject(documentType, union.SubType, seen, survey); + { + graph.Edges.Add((type, null, union.SubType, Array.Empty())); + VisitObject(documentType, union.SubType, seen, survey, graph); + } } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, Survey survey) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; - if (IsLeaf(memberType)) + // A sequence or map whose element is itself (an unkeyed Node : IEnumerable) ends the type walk here; without + // this it would recurse until the stack overflows. + if (Array.IndexOf(through, memberType) >= 0) return; - if (DictionaryValueType(memberType) is { } valueType) + var (kind, inner) = Classify(memberType); + switch (kind) { - VisitMemberType(documentType, owner, member, valueType, seen, survey); - return; - } - - if (ElementTypeOf(memberType) is { } element) - { - if (IsObjectType(element)) - { - foreach (var concrete in ConcreteTypes(element)) + case Kind.Leaf: + // A shape the walk does not own is one whole value, never refused (q3 amended). + return; + case Kind.Map: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + case Kind.Sequence when IsElement(inner): + var concretes = ConcreteTypes(inner).ToList(); + graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray())); + foreach (var concrete in concretes) { - var problem = IdProblem(concrete); + if (!survey.Judged.Add(concrete)) + continue; + var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; if (problem != null) survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); } - VisitObject(documentType, element, seen, survey); + VisitObject(documentType, inner, seen, survey, graph); + return; + case Kind.Sequence: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + default: + graph.Edges.Add((owner, member, memberType, through)); + VisitObject(documentType, memberType, seen, survey, graph); + return; + } + } + + private const string UnkeyedElementProblem = + "is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists."; + + // The id path: every type that reaches an object list through members MessagePack writes by key, the lists' element and + // concrete types, and the keyed types that enumerate objects. The codec verifies that MessagePack writes each of them the + // way Classify reads it; it never decides ownership. + private static void Settle(Type documentType, Survey survey, Graph graph) + { + var lists = graph.Lists; + var reaching = new HashSet(lists.Select(list => list.Owner)); + var through = new List(lists.SelectMany(list => list.Through)); + for (var grew = true; grew;) + { + grew = false; + foreach (var (owner, _, child, via) in graph.Edges) + { + if (!reaching.Contains(child)) + continue; + through.AddRange(via); + grew |= reaching.Add(owner); } - else - VisitMemberType(documentType, owner, member, element, seen, survey); - return; } - if (IsObjectType(memberType)) - VisitObject(documentType, memberType, seen, survey); + var path = new List(); + void Add(Type type) + { + if (!path.Contains(type)) + path.Add(type); + } + + foreach (var type in reaching) + Add(type); + foreach (var type in through) + Add(type); + foreach (var list in lists) + { + Add(list.Element); + foreach (var concrete in list.Concrete) + Add(concrete); + } + + // A keyed type the walk reads by key but whose enumeration holds objects is on the path: the codec asks whether + // MessagePack writes it by key too. + foreach (var type in graph.Enumerating) + Add(type); + + survey.IdPath.AddRange(path); } // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype @@ -171,21 +259,31 @@ namespace GameCult.Caching private static string? IdProblem(Type element) { - var ids = MembersOf(element).Where(member => member.IsDefined(typeof(CultElementIdAttribute), true)).ToArray(); + // Candidates come from every instance member up the base chain, public or not, so an id MessagePack does not write + // is named as such instead of being reported missing. An override or a hiding member is one member, the most derived. + var ids = AllInstanceMembers(element).Where(member => Declares(member, typeof(CultElementIdAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()).ToArray(); if (ids.Length == 0) return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity."; if (ids.Length > 1) return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity."; var id = ids[0]; + if (Declares(id, typeof(MessagePackFormatterAttribute))) + return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key]."; if (TypeOf(id) != typeof(string)) return $"declares [CultElementId] on {id.Name}, which is not a string."; if (KeyOf(id) is not { } idKey) return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted."; + if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written) + return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], or non-public on a type without [MessagePackObject(AllowPrivate = true)]); the id would never reach disk."; + id = written; var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey); if (sharing.Member != null) return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own."; if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true }) return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it."; + if (id is PropertyInfo { SetMethod.IsPublic: false } && !AllowsPrivate(element)) + return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine)."; var source = id.GetCustomAttribute(true)!.DerivedFrom; if (source != null) { @@ -203,40 +301,37 @@ namespace GameCult.Caching // ---- write: what the stored form holds ---- // True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value - // as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. - internal static bool Holds(object? root) + // as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. The walk + // follows declared types, as registration does; a value's own type picks only a union arm. + internal static bool Holds(Type declared, object? root) { if (root == null) return false; - var type = root.GetType(); - if (IsLeaf(type)) - return false; - if (root is IDictionary dictionary) + var (kind, inner) = Classify(declared); + switch (kind) { - foreach (DictionaryEntry entry in dictionary) - { - if (Holds(entry.Value)) - return true; - } + case Kind.Leaf: + return false; + case Kind.Map: + return MapEntries(declared)(root).Any(entry => Holds(inner, entry.Value)); + case Kind.Sequence: + var elements = IsElement(inner); + if (!elements && Classify(inner).Kind == Kind.Leaf) + return false; + foreach (var item in (IEnumerable)root) + { + if (item == null) + continue; + if (elements && OwnerShape(inner, item).GetId is { } getId && !string.IsNullOrEmpty(getId(item))) + return true; + if (Holds(inner, item)) + return true; + } - return false; + return false; + default: + return OwnerShape(declared, root).Walk.Any(member => Holds(member.Type, member.Get(root))); } - - if (root is IEnumerable list) - { - foreach (var item in list) - { - if (item != null && IsObjectType(item.GetType()) && ShapeOf(item.GetType()).GetId is { } getId && - !string.IsNullOrEmpty(getId(item))) - return true; - if (Holds(item)) - return true; - } - - return false; - } - - return IsObjectType(type) && ShapeOf(type).Walk.Any(member => Holds(member.Get(root))); } // ---- write and load: mint and check ---- @@ -245,41 +340,48 @@ namespace GameCult.Caching // that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty. // rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits // twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load. - internal static IdPlan Plan(object? root, string rootPath, bool deterministic) + internal static IdPlan Plan(Type declared, object? root, string rootPath, bool deterministic) { var plan = new IdPlan(); - Visit(root, rootPath); + Visit(declared, root, rootPath); return plan; - void Visit(object? value, string path) + void Visit(Type type, object? value, string path) { if (value == null) return; - var type = value.GetType(); - if (IsLeaf(type)) - return; - if (value is IDictionary dictionary) + var (kind, inner) = Classify(type); + switch (kind) { - foreach (DictionaryEntry entry in dictionary) - Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); - } - else if (value is IEnumerable list) - Elements(list, path); - else if (IsObjectType(type)) - { - foreach (var (slot, get) in ShapeOf(type).Walk) - Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); + case Kind.Map: + foreach (var entry in MapEntries(type)(value)) + Visit(inner, entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); + break; + case Kind.Sequence when IsElement(inner): + Elements(inner, (IEnumerable)value, path); + break; + case Kind.Sequence: + if (Classify(inner).Kind == Kind.Leaf) + break; + var index = 0; + foreach (var item in (IEnumerable)value) + Visit(inner, item, path + "/" + (index++).ToString(CultureInfo.InvariantCulture)); + break; + case Kind.Owner: + foreach (var (slot, memberType, get) in OwnerShape(type, value).Walk) + Visit(memberType, get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); + break; } } - void Elements(IEnumerable list, string path) + void Elements(Type element, IEnumerable list, string path) { var items = list.Cast().ToArray(); var taken = new HashSet(StringComparer.Ordinal); var objects = new HashSet(ReferenceComparer.Instance); foreach (var item in items) { - if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape) + if (item == null || OwnerShape(element, item) is not { GetId: { } getId } shape) continue; if (!objects.Add(item)) throw new CultElementIdException( @@ -302,7 +404,7 @@ namespace GameCult.Caching if (item == null) continue; var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture); - if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape && + if (OwnerShape(element, item) is { SetId: { } , GetId: { } get } itemShape && string.IsNullOrEmpty(get(item))) { string id; @@ -321,7 +423,7 @@ namespace GameCult.Caching plan.Add(item, itemShape, get(item), id); } - Visit(item, itemPath); + Visit(element, item, itemPath); } } @@ -389,10 +491,18 @@ namespace GameCult.Caching private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build); + // The shape of an owner or element a walk holds under a declared type: a union base writes its arm, so the value's own + // type picks the arm; any other declared type is written by its own [Key] layout, whatever a subclass adds. + private static Shape OwnerShape(Type declared, object value) + { + declared = Nullable.GetUnderlyingType(declared) ?? declared; + return declared.IsDefined(typeof(UnionAttribute), false) ? ShapeOf(value.GetType()) : ShapeOf(declared); + } + private static Shape Build(Type type) { var shape = new Shape(); - var idMember = MembersOf(type).FirstOrDefault(member => member.IsDefined(typeof(CultElementIdAttribute), true)); + var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute))); if (idMember != null) { shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o); @@ -406,20 +516,62 @@ namespace GameCult.Caching } shape.Walk = KeyedMembers(type) - .Where(entry => !IsLeaf(Nullable.GetUnderlyingType(entry.Type) ?? entry.Type)) - .Select(entry => (Slot: KeyOf(entry.Member)!.Value, Get: entry.Member is FieldInfo fi + .Where(entry => Classify(entry.Type).Kind != Kind.Leaf) + .Select(entry => (Slot: KeyOf(entry.Member)!.Value, entry.Type, Get: entry.Member is FieldInfo fi ? (Func)(o => fi.GetValue(o)) : o => ((PropertyInfo)entry.Member).GetValue(o))) .ToArray(); return shape; } + // The members MessagePack's StandardResolver writes for an integer-keyed type, none of them [IgnoreMember]. Without + // AllowPrivate: public instance fields, and public instance properties with a public getter. With it, read from the most + // derived [MessagePackObject] declaration (a derived type without it is written without it, probe NoApDerived): every + // instance field and indexer-free property up the base chain at any visibility, the most derived of a name. + // TheWalkReadsTheMembersMessagePackWrites checks this set against MessagePack's own member discovery. private static IEnumerable MembersOf(Type type) => + AllowsPrivate(type) + ? AllInstanceMembers(type).Where(member => !Declares(member, typeof(IgnoreMemberAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()) + : PublicMembersOf(type); + + private static bool AllowsPrivate(Type type) => type.GetCustomAttribute(true)?.AllowPrivate == true; + + private static IEnumerable PublicMembersOf(Type type) => type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast() - .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod != null)); + .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + .Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true })) + .Where(member => !Declares(member, typeof(IgnoreMemberAttribute))); + + // Every instance field and indexer-free property declared on the type or any base, most derived first. + private static IEnumerable AllInstanceMembers(Type type) + { + const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + for (var level = type; level != null && level != typeof(object); level = level.BaseType) + { + foreach (var field in level.GetFields(flags)) + yield return field; + foreach (var property in level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0)) + yield return property; + } + } + + // The [Key] members MessagePack writes, for the oracle test that checks them against MessagePack. + internal static IEnumerable WrittenKeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null); private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) => - MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member))); + WrittenKeyedMembers(type).Where(member => !Declares(member, typeof(MessagePackFormatterAttribute))) + .Select(member => (member, TypeOf(member))); + + // An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an + // override of a base virtual member would not see its base's attribute. + private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); + + // True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it. + internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner; + + private static bool EnumeratesObjects(Type type) => + (MapTypes(type)?.Value ?? ElementTypeOf(type)) is { } inner && Classify(inner).Kind != Kind.Leaf; private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; @@ -427,18 +579,57 @@ namespace GameCult.Caching // ---- type classification ---- - // A value MessagePack writes whole and that cannot hold an element list. Structs are values: never elements, never walked. - private static bool IsLeaf(Type type) - { - if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type.IsValueType) - return true; - return type.IsArray && IsLeaf(type.GetElementType()!); - } + private enum Kind { Leaf, Map, Sequence, Owner } - // An object a store can hold an element list under, or hold as an element: a class MessagePack serializes by key. - private static bool IsObjectType(Type type) => - (type.IsClass || type.IsInterface) && type != typeof(string) && type != typeof(object) && - (type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true)); + private static readonly ConcurrentDictionary Classes = new(); + + // The walk's model of a declared type: registration, Plan, Holds and the codec's id path all go through it. A + // [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable; the codec checks, for every type + // on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not. + // Inner is a map's value type, a sequence's element type, or the type itself for an owner. + // The owned shapes are a closed list (Owner, Sequence, Map); every other shape is Leaf: one whole value with no element + // ids, replaced whole by a variant (q3 amended 2026-10-16). When in doubt the answer is Leaf. + // Leaf strings (an IEnumerable MessagePack writes whole), a type written whole (IsWrittenWhole: attribute- + // formatted, [DataContract], string-keyed), a struct or a class or interface with no [MessagePackObject] or + // [Union] and no collection shape (primitives, enums, float2, DateTime, ImmutableArray, ArraySegment, Memory, + // KeyValuePair, tuples, Lazy, unkeyed classes), and object, which is untyped on disk. + // Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>. + // Sequence an array or an IEnumerable class or interface. + // Owner a class, interface or struct MessagePack serializes by integer key. A keyed struct is walked as an owner; + // it is never an element (IsElement), since a boxed copy cannot carry a minted id. + private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type => + { + type = Nullable.GetUnderlyingType(type) ?? type; + if (type == typeof(string)) + return (Kind.Leaf, type); + if (IsWrittenWhole(type)) + return (Kind.Leaf, type); + if (IsKeyed(type)) + return (Kind.Owner, type); + if (type.IsValueType) + return (Kind.Leaf, type); + if (MapTypes(type) is { } map) + return (Kind.Map, map.Value); + if (ElementTypeOf(type) is { } element) + return (Kind.Sequence, element); + return (Kind.Leaf, type); + }); + + // A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type, + // inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string + // keys (KeyAsPropertyName, or a [Key] with a string key on a member MessagePack writes: a private or [IgnoreMember] one is not written). Such a type is Leaf, even when it is also enumerable. + private static bool IsWrittenWhole(Type type) => + type.IsDefined(typeof(MessagePackFormatterAttribute), true) || + (type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) || + type.GetCustomAttribute(true)?.KeyAsPropertyName == true || + MembersOf(type).Any(member => member.GetCustomAttribute(true)?.StringKey != null); + + // A type MessagePack serializes by key: [MessagePackObject] or [Union]. + private static bool IsKeyed(Type type) => + type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true); + + // What an object list holds: a keyed class or interface. Never a struct. + private static bool IsElement(Type type) => !type.IsValueType && Classify(type).Kind == Kind.Owner; private static Type? ElementTypeOf(Type type) { @@ -453,15 +644,34 @@ namespace GameCult.Caching return null; } - private static Type? DictionaryValueType(Type type) + // The key and value types of a map MessagePack writes as a map: IDictionary<,> or IReadOnlyDictionary<,>. + private static (Type Key, Type Value)? MapTypes(Type type) { foreach (var candidate in type.GetInterfaces().Append(type)) { - if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>)) - return candidate.GetGenericArguments()[1]; + if (candidate.IsGenericType && + (candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>) || candidate.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>))) + { + var arguments = candidate.GetGenericArguments(); + return (arguments[0], arguments[1]); + } } return null; } + + private static readonly ConcurrentDictionary>> MapReaders = new(); + + // A map's entries read through the generic interface Classify found, so no map shape is recognised twice. + private static Func> MapEntries(Type map) => MapReaders.GetOrAdd(map, static type => + { + var (key, value) = MapTypes(type)!.Value; + return (Func>)Delegate.CreateDelegate( + typeof(Func>), + typeof(CultElementIds).GetMethod(nameof(ReadMap), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(key, value)); + }); + + private static IEnumerable<(object? Key, object? Value)> ReadMap(object map) => + ((IEnumerable>)map).Select(entry => ((object?)entry.Key, (object?)entry.Value)); } } diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index 6e47742f..fc5cd308 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -195,6 +195,15 @@ A document type that holds a list of objects gives every object type in that lis `[Key]`. The id is what lets a variant override one list element instead of the whole list. Lists of strings, numbers and other non-object values need none. +The rule covers the shapes the id walk owns: arrays, `IEnumerable` classes and +interfaces, dictionary values, and integer-keyed `[MessagePackObject]` and `[Union]` +types. Any other holder is one whole value: `ImmutableArray`, `ArraySegment`, +`Memory`, `KeyValuePair`, tuples, `Lazy`, a class without `[MessagePackObject]`, +a `[DataContract]` or string-keyed type, and a type or member declaring +`[MessagePackFormatter]`. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked, and a variant can only +replace the holder whole. + ```csharp [MessagePackObject] public sealed class Line diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 495b2273..40480895 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -98,7 +98,7 @@ namespace GameCult.Mesh var descriptor = resolvedRegistry.GetRequired(); // Element ids are decided as a cache write decides them, before anything is written: an unset id is minted, a // refused list refuses the write, and the file is marked when it holds an id. A failed write gives the ids back. - var ids = CultElementIds.Plan(document, key.Value, deterministic: false); + var ids = CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: false); ids.Apply(); try { @@ -108,7 +108,8 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - CultElementIds.Holds(document)); + CultElementIds.Holds(typeof(TDocument), document), + resolvedRegistry); } catch { @@ -131,7 +132,7 @@ namespace GameCult.Mesh var payload = ReadSingleFileDocumentPayload(path, key, descriptor.SchemaId); var document = (TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), payload); // A load refuses what a write refuses, and mints the ids a cache load of the same file would mint. - CultElementIds.Plan(document, key.Value, deterministic: true).Apply(); + CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: true).Apply(); return document; } @@ -146,7 +147,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null); } /// @@ -229,7 +230,8 @@ namespace GameCult.Mesh CultSchemaCatalogEntry catalogEntry, string? storedAt, byte[] payload, - bool? holdsIds) + bool? holdsIds, + CultDocumentRegistry? registry) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -239,8 +241,13 @@ namespace GameCult.Mesh payload ??= Array.Empty(); // The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under // the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null), - // so a file already marked stays marked. A store this runtime cannot read refuses the write instead of being overwritten. - StoreAt(path).ApplyWriteSet( + // so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write + // decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime + // cannot read refuses the write instead of being overwritten. + var store = StoreAt(path); + if (registry != null) + store.AttachRegistry(registry); + store.ApplyWriteSet( new[] { new CultStagedRecord( diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 0dd8b125..708f0a09 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -6,7 +6,10 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(PairResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 7ae2ee20..2bc1d9df 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -1,7 +1,9 @@ #nullable enable using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; +using System.Reflection; using System.Text.RegularExpressions; using GameCult.Caching; using MessagePack; @@ -16,7 +18,7 @@ namespace GameCult.Caching.Tests private static int Assign(object root, string key, bool deterministic) { - var plan = CultElementIds.Plan(root, key, deterministic); + var plan = CultElementIds.Plan(root.GetType(), root, key, deterministic); plan.Apply(); return plan.Count; } @@ -30,7 +32,8 @@ namespace GameCult.Caching.Tests foreach (var holder in new[] { typeof(HList), typeof(HArray), typeof(HDictionary), typeof(HNested), typeof(HListOfLists), - typeof(HElementNest), typeof(HUnion), typeof(HNullableInner) + typeof(HElementNest), typeof(HUnion), typeof(HNullableInner), typeof(HStruct), typeof(HStructList), typeof(HNullableStruct), + typeof(HReadOnlyMap) }) RefusesBare(holder); Assert.That(Problems(typeof(HFine)), Is.Empty); @@ -72,28 +75,28 @@ namespace GameCult.Caching.Tests public void HoldsFindsASetIdWhereverAListCanSitAndNothingElse() { const string id = "0123456789ab"; - Assert.That(CultElementIds.Holds(null), Is.False); - Assert.That(CultElementIds.Holds(new HAll + Assert.That(CultElementIds.Holds(typeof(HAll), null), Is.False); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded() } } }, Grid = { new List { new EIded() } }, Nest = { new ENest { Sub = { new EIded() } } }, }), Is.False, "unset ids hold nothing"); - Assert.That(CultElementIds.Holds(new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); - Assert.That(CultElementIds.Holds(new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); - Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); - Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); } [Test] public void APlanCountsAndChangesNothingUntilApplied() { var root = new HHasIds { Items = { new EIded(), new EIded() } }; - var plan = CultElementIds.Plan(root, "r", deterministic: true); + var plan = CultElementIds.Plan(typeof(HHasIds), root, "r", deterministic: true); Assert.That(plan.Count, Is.EqualTo(2)); Assert.That(root.Items.Select(x => x.Id), Has.All.Empty); plan.Apply(); @@ -184,7 +187,7 @@ namespace GameCult.Caching.Tests Assert.That(root.Dict["k"].Items.Select(x => x.Id), Has.All.Empty, "the list before the refused one was not minted into"); var ok = new HHasIds { Items = { new EIded(), new EIded { Id = "dddddddddddd" } } }; - var plan = CultElementIds.Plan(ok, "r", deterministic: false); + var plan = CultElementIds.Plan(typeof(HHasIds), ok, "r", deterministic: false); plan.Apply(); plan.Undo(); Assert.That(ok.Items.Select(x => x.Id), Is.EqualTo(new[] { "", "dddddddddddd" })); @@ -290,12 +293,637 @@ namespace GameCult.Caching.Tests Assert.That(Assert.Throws(() => Assign(twice, "r", deterministic: false))!.Message, Does.Contain("'5'")); } + // A list element is written by its [Key] layout or its id never reaches disk. A union arm without [MessagePackObject] + // is written by some other formatter; a list of a type that is not keyed at all is a value list, replaced whole. + [Test] + public void ANonKeyedArmOfAListUnionIsRefusedAtRegistration() + { + var problem = Problems(typeof(HUnkeyedArmList)).Single(); + Assert.That(problem, Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own").And.Contain("HUnkeyedArmList")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty, "the same union as a field has no element id to lose"); + Assert.That(Problems(typeof(HValueList)), Is.Empty, "a list of a non-keyed type is a value list"); + } + + // [MessagePackFormatter] on an element type, an owner, a collection class or a member makes that value one whole value: it + // has no element ids and is off the id path. The attribute on the id member itself is the one refusal that stays. + [Test] + public void AFormatterAttributeMakesAWholeValueExceptOnTheId() + { + Assert.That(Problems(typeof(HFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + foreach (var (type, holder) in new (Type, object)[] + { + (typeof(HFormattedElement), new HFormattedElement { Items = { new FormattedElement() } }), + (typeof(HFormattedList), new HFormattedList { Items = { new EIded() } }), + (typeof(HFormattedOwner), new HFormattedOwner { Owner = { Items = { new EIded() } } }), + (typeof(HFormattedMember), new HFormattedMember { Inner = { Items = { new EIded() } } }), + (typeof(HFormattedDerived), new HFormattedDerived { Items = { new FormattedDerived() } }), + }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(0), type.Name); + } + + Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field"); + } + + // The collection types between a member and its object list are written by some formatter too, so they sit on the id path. + // A value list (of a struct, a primitive or a non-keyed class) never reaches an object, so it does not. + [Test] + public void ACollectionTypeOnTheWayToAnObjectListIsOnTheIdPath() + { + var path = CultElementIds.Inspect(typeof(HCollections)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] + { + typeof(HCollections), typeof(EIded), typeof(HHasIds), + typeof(List), typeof(EIded[]), typeof(Dictionary), typeof(List>) + })); + + var valueLists = CultElementIds.Inspect(typeof(HValueListsBesideObjects)).IdPath; + Assert.That(valueLists, Is.EquivalentTo(new[] { typeof(HValueListsBesideObjects), typeof(List), typeof(EIded) })); + } + + [Test] + public void AFormatterOnACollectionClassOfAnObjectListIsAWholeValue() + { + Assert.That(Problems(typeof(HRoles)), Is.Empty); + Assert.That(Assign(new HRoles { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "its elements carry no ids and are never minted"); + Assert.That(Problems(typeof(HFormattedValueCollection)), Is.Empty, "a collection class of a non-keyed element is a value list"); + } + + // MemberInfo.IsDefined ignores inherit for properties: an override sees its base's attribute only through Declares. + [Test] + public void AnInheritedFormatterOnAnOverriddenMemberIsHonoured() + { + Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(OwnerOverFormatted)), Is.Empty, "the override inherits its base's formatter, so the member is a whole value"); + Assert.That(CultElementIds.Inspect(typeof(OwnerOverFormatted)).IdPath, Is.Empty, "a walked override would put its list on the id path"); + Assert.That(Assign(new OwnerOverFormatted { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "and mint into a list its formatter may drop ids from"); + + Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id"); + var root = new HIdOverride { Items = { new IdOverride() } }; + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(root.Items[0].Id, Has.Length.EqualTo(12)); + } + + // The union arm has a valid id, so IdProblem passes it and only the rule that an element is written by its [Key] layout + // can refuse it; it is refused once per document however many lists hold it. + [Test] + public void ANonKeyedArmWithAValidIdIsRefusedOncePerType() + { + Assert.That(Problems(typeof(HUnkeyedArmList)).Single(), Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own")); + Assert.That(Problems(typeof(HUnkeyedTwoLists)).Single(), Does.Contain("UnkeyedArm")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty); + } + + // The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and + // a type reached twice, are both on the way to it. + [Test] + public void TheIdPathHoldsEveryTypeThatReachesAnObjectList() + { + var path = CultElementIds.Inspect(typeof(HDiamond)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded), typeof(List), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty); + + // A union base is on the path both as the type a list holds (its formatter writes the arm) and as a field type whose + // arm holds the list. + Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded), typeof(List) })); + } + + // A keyed struct is a value MessagePack writes by key, and an object list inside one carries element ids like any other. + // It is walked as an owner and is never an element: a boxed copy cannot carry a minted id. + [Test] + public void AKeyedStructIsWalkedAsAnOwnerAndNeverAnElement() + { + var root = new HStructIded + { + Hold = new SHoldIded { Items = new List { new EIded() } }, + Many = { new SHoldIded { Items = new List { new EIded() } } }, + }; + Assert.That(Problems(typeof(HStructIded)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructIded)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructIded), typeof(SHoldIded), typeof(List), typeof(List), typeof(EIded) })); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.False); + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(root.Hold.Items[0].Id, Is.Not.Empty, "visible through the struct copy"); + Assert.That(root.Many[0].Items[0].Id, Is.Not.Empty, "visible through the struct in a list"); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.True); + var one = new HStructIded { Hold = new SHoldIded { Items = new List { new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Holds(typeof(HStructIded), one), Is.True); + + Assert.That(Problems(typeof(HStructValues)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructValues)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructValues), typeof(List), typeof(EIded) }), "a list of keyed structs is a value list"); + var values = new HStructValues { Structs = { new SValue { N = 1 } }, Objects = { new EIded() } }; + Assert.That(Assign(values, "r", deterministic: false), Is.EqualTo(1)); + + var nullable = new HNullableStructIded { Hold = new SHoldIded { Items = new List { new EIded() } } }; + Assert.That(Assign(nullable, "r", deterministic: false), Is.EqualTo(1), "a nullable struct is walked as its struct"); + Assert.That(CultElementIds.Holds(typeof(HNullableStructIded), nullable), Is.True); + } + + // A type is an owner or an element only when MessagePack serializes it by key ([MessagePackObject], inherited, or + // [Union]). A struct without either is a value however many [Key] members it carries. + [Test] + public void OnlyAKeyedTypeIsWalked() + { + Assert.That(Problems(typeof(HUnkeyedStruct)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HUnkeyedStruct)).IdPath, Is.Empty); + Assert.That(Problems(typeof(HInheritedBare)).Single(), Does.Contain("InheritedBare").And.Contain("[CultElementId]"), + "[MessagePackObject] reaches a subclass through inheritance"); + } + + // Every dictionary interface MessagePack writes as a map is walked by one path, and the minted path text is the one a + // store loaded before ids already holds, so its deterministic ids do not move. + [Test] + public void EveryMapInterfaceIsWalkedByTheSamePath() + { + var element = new Func(() => new HHasIds { Items = { new EIded() } }); + var viaInterface = new HReadOnlyIded { Map = new Dictionary { ["k"] = element() } }; + var viaClass = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = element() }) }; + var viaDictionary = new HMapIded { Map = { ["k"] = element() } }; + Assert.That(Assign(viaInterface, "r", deterministic: true), Is.EqualTo(1)); + Assert.That(Assign(viaClass, "r", deterministic: true), Is.EqualTo(1), "a class implementing only IReadOnlyDictionary<,>"); + Assert.That(Assign(viaDictionary, "r", deterministic: true), Is.EqualTo(1)); + var id = viaDictionary.Map["k"].Items[0].Id; + Assert.That(viaInterface.Map["k"].Items[0].Id, Is.EqualTo(id)); + Assert.That(viaClass.Map["k"].Items[0].Id, Is.EqualTo(id)); + + // SHA-256 of "||0": slot 0, the key in braces, slot 0, the index. + var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("r|r.0{k}.0/0|0")); + Assert.That(id, Is.EqualTo(Convert.ToHexString(hash, 0, 6).ToLowerInvariant()), "the path text is unchanged"); + + var held = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = new HHasIds { Items = { new EIded { Id = "0123456789ab" } } } }) }; + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), held), Is.True); + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), viaInterface), Is.True); + Assert.That(CultElementIds.Holds(typeof(HMapIded), new HMapIded { Map = { ["k"] = new HHasIds { Items = { new EIded() } } } }), Is.False); + } + + // Registration judges the declared shape, so minting and reading follow it: a member declared object is untyped on disk, + // and what a subclass adds under a base-typed member is not written, so neither is walked. + [Test] + public void TheWalkFollowsTheDeclaredShape() + { + var derived = new HBase { B = new DerC { L = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HBase), derived, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HBase), derived), Is.False); + var opaque = new HObject { O = new HHasIds { Items = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HObject), opaque, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HObject), opaque), Is.False); + foreach (var type in new[] { typeof(HBase), typeof(HObject) }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + // A union base still writes its arm, so the arm is walked. + var arm = new HIdBaseOwner { Root = new IdArm { Items = { new EIded() } } }; + Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1)); + } + + // MessagePack writes a non-public [Key] member only under AllowPrivate, so a non-public member of a type without it is nobody's + // id business; under AllowPrivate it is written, read and minted like any other member. + [Test] + public void ANonPublicMemberIsWalkedOnlyUnderAllowPrivate() + { + foreach (var type in new[] { typeof(HHiddenList), typeof(HHiddenOwner), typeof(HHiddenFromBase) }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + Assert.That(Problems(typeof(HOptHidden)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HOptHidden)).IdPath, Is.EquivalentTo(new[] { typeof(HOptHidden), typeof(List), typeof(EIded) })); + var opt = new HOptHidden(); + opt.Peek.Add(new EIded()); + opt.Peek.Add(new EIded()); + Assert.That(Assign(opt, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(opt.Peek.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); + + Assert.That(Problems(typeof(HHiddenLeaf)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HHiddenLeaf)).IdPath, Is.EquivalentTo(new[] { typeof(HHiddenLeaf), typeof(List), typeof(EIded) }), + "a non-public member adds nothing to the id path"); + } + + // MessagePack writes a [MessagePackObject] class that is only IEnumerable by its keys, so the walk reads it by key. A + // keyed tree node that enumerates its own children is the shape that overflowed the stack when enumeration came first. + [Test] + public void ATypeThatEnumeratesItselfIsWalkedByKeyAndTerminates() + { + var survey = CultElementIds.Inspect(typeof(HKNode)); + Assert.That(survey.Problems, Is.Empty); + Assert.That(survey.IdPath, Is.EquivalentTo(new[] { typeof(HKNode), typeof(List), typeof(KNode) })); + + var root = new HKNode { Nodes = { new KNode { Children = { new KNode { Children = { new KNode() } } } } } }; + Assert.That(Assign(root, "r", deterministic: true), Is.EqualTo(3)); + Assert.That(root.Nodes[0].Id, Has.Length.EqualTo(12)); + Assert.That(root.Nodes[0].Children[0].Children[0].Id, Has.Length.EqualTo(12)); + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(root, options), options); + Assert.That(back.Nodes[0].Id, Is.EqualTo(root.Nodes[0].Id), "MessagePack writes the node by key, ids included"); + Assert.That(back.Nodes[0].Children[0].Children[0].Id, Is.EqualTo(root.Nodes[0].Children[0].Children[0].Id)); + + // An unkeyed type that enumerates itself holds no element list and ends the type walk. + var unkeyed = CultElementIds.Inspect(typeof(HUNode)); + Assert.That(unkeyed.Problems, Is.Empty); + Assert.That(unkeyed.IdPath, Is.Empty); + + // A keyed class that only enumerates is read by its keyed lists, so both are judged and minted. + Assert.That(Problems(typeof(KBag)).Single(), Does.Contain("EBare")); + var bag = new KBagIded { Items = { new EIded(), new EIded() } }; + Assert.That(Assign(bag, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True); + } + + // The walk reads the members MessagePack writes: without AllowPrivate a public field or a property with a public getter, with + // it every member up the base chain, the most derived [MessagePackObject] declaration deciding; no [IgnoreMember]. + [Test] + public void TheWalkReadsTheMembersMessagePackWrites() + { + Assert.That(Problems(typeof(PubIgn)), Is.Empty, "an ignored member is never written, so its list is nobody's business"); + Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written"); + Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate"); + + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor), typeof(ApIgnAp), typeof(PSkH), typeof(ISkH) }) + { + var allowPrivate = type.GetCustomAttribute(true)!.AllowPrivate; + Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), + Is.EquivalentTo(MessagePackMembers(type, allowPrivate)), type.Name); + } + + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" }), + "the derived declaration decides, so an AllowPrivate base adds nothing"); + } + + // The names of the members MessagePack's own discovery reads for a type, by reflection into its internal type: the + // oracle the walk's member set is pinned to. + private static List MessagePackMembers(Type type, bool allowPrivate) + { + const string moved = "MessagePack moved its member discovery; re-derive MembersOf"; + const System.Reflection.BindingFlags all = System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic; + var info = typeof(MessagePackSerializer).Assembly.GetType("MessagePack.Internal.ObjectSerializationInfo"); + var create = info?.GetMethod("CreateOrNull", all); + if (create == null || create.GetParameters().Length != 4) + { + Assert.Fail(moved); + return new List(); + } + + var result = create.Invoke(null, new object[] { type, false, false, allowPrivate }); + var members = result?.GetType().GetProperty("Members", all)?.GetValue(result) as System.Collections.IEnumerable; + if (members == null) + { + Assert.Fail(moved); + return new List(); + } + + var names = new List(); + foreach (var member in members) + { + var readable = member.GetType().GetProperty("IsReadable", all)?.GetValue(member); + var name = member.GetType().GetProperty("Name", all)?.GetValue(member) as string; + if (readable is not bool || name == null) + Assert.Fail(moved); + if ((bool)readable!) + names.Add(name!); + } + + return names; + } + + // An id MessagePack writes but cannot read back would be re-minted differently on every load; one it does not write + // never reaches disk. + [Test] + public void AnIdMessagePackCannotReadBackIsRefused() + { + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id")); + Assert.That(Problems(typeof(HIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HPrivateId)).Single(), Does.Contain("does not write").And.Contain("id")); + Assert.That(Problems(typeof(HInit)), Is.Empty); + Assert.That(Problems(typeof(HCtorId)), Is.Empty); + + foreach (var holder in new object[] + { + new HInit { Items = { new EInit(), new EInit() } }, + new HCtorId { Items = { new ECtorId(null), new ECtorId(null) } }, + }) + { + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + var bytes = MessagePackSerializer.Serialize(holder.GetType(), holder, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + var back = MessagePackSerializer.Deserialize(holder.GetType(), bytes, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + Assert.That(Ids(back), Is.EqualTo(Ids(holder)), holder.GetType().Name); + Assert.That(Ids(back), Has.All.Match("^[0-9a-f]{12}$"), holder.GetType().Name); + } + } + + // Every shape the walk does not own is one whole value: no problem, off the id path, never minted into and never counted. + [Test] + public void AShapeTheWalkDoesNotOwnIsOneWholeValue() + { + const string Set = "aaaaaaaaaaaa"; + EIded E(string id) => new() { Id = id }; + var holders = new (Type Type, Func Make)[] + { + (typeof(Whole>), id => new Whole> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole?>), id => new Whole?> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole>), id => new Whole> { Value = new ArraySegment(new[] { E(id) }) }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>>), id => new Whole>> { Value = new("k", new List { E(id) }) }), + (typeof(Whole<(List, int)>), id => new Whole<(List, int)> { Value = (new List { E(id) }, 1) }), + (typeof(Whole, int>>), id => new Whole, int>> { Value = Tuple.Create(new List { E(id) }, 1) }), + (typeof(Whole>>), id => new Whole>> { Value = new Lazy>(() => new List { E(id) }) }), + (typeof(Whole), id => new Whole { Value = new NkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new SkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new DcHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpBase { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpDerived { Items = { E(id) } } }), + (typeof(HRoles), id => new HRoles { Items = { E(id) } }), + (typeof(HFormattedList), id => new HFormattedList { Items = { E(id) } }), + (typeof(HHiddenList), id => new HHiddenList()), + }; + foreach (var (type, make) in holders) + { + Assert.That(Problems(type), Is.Empty, type.ToString()); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.ToString()); + Assert.That(CultElementIds.CanHoldIds(type), Is.False, type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: false), Is.EqualTo(0), type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: true), Is.EqualTo(0), type.ToString()); + Assert.That(CultElementIds.Holds(type, make(Set)), Is.False, type.ToString()); + } + + // A [DataContract] subclass of a [MessagePackObject] type is still read by key: the attribute it inherits keeps it an owner. + Assert.That(CultElementIds.Inspect(typeof(Whole)).IdPath, Does.Contain(typeof(List))); + + // Elements of a string-keyed type carry no id either, and an owned list of a bare element is still refused. + Assert.That(Problems(typeof(Whole>)), Is.Empty); + RefusesBare(typeof(Whole>)); + } + + // A string key counts only on a member MessagePack writes: a private or [IgnoreMember] one leaves the type integer-keyed and owned. + [Test] + public void AStringKeyOnAMemberMessagePackDoesNotWriteLeavesTheTypeOwned() + { + foreach (var holder in new object[] { new PSkH { Items = { new EIded(), new EIded() } }, new ISkH { Items = { new EIded(), new EIded() } } }) + { + Assert.That(Problems(holder.GetType()), Is.Empty, holder.GetType().Name); + Assert.That(CultElementIds.Inspect(holder.GetType()).IdPath, Does.Contain(holder.GetType()), holder.GetType().Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + Assert.That(CultElementIds.Holds(holder.GetType(), holder), Is.True, holder.GetType().Name); + } + + Assert.That(CultElementIds.WalksAsOwner(typeof(SkOver)), Is.False, "a string key inherited by an override still makes the type one whole value"); + Assert.That(CultElementIds.WalksAsOwner(typeof(PSkH)), Is.True); + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]")); + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]"), "an element whose only string key is ignored is as bare as its twin"); + } + + // The AllowPrivate edges: an [IgnoreMember] member is not written even under it, and only the element's own most derived + // declaration lets a private setter be read back. + [Test] + public void TheAllowPrivateEdgesFollowWhatMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApIgnAp)), Is.Empty, "an ignored member is not written under AllowPrivate"); + Assert.That(Problems(typeof(HApIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HNoApPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id"), + "the derived plain [MessagePackObject] decides: its private setter is not read back, whatever its base declares"); + } + + // MessagePack writes an AllowPrivate type with every member up its base chain, and the most derived [MessagePackObject] + // declaration decides whether that is so. + [Test] + public void AnAllowPrivateTypeIsWalkedByTheMembersMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApInheritor)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" }), + "AllowPrivate is inherited by a subclass that declares no attribute of its own"); + + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var held = new HOptHidden(); + held.Peek.Add(new EIded()); + Assert.That(Assign(held, "r", deterministic: false), Is.EqualTo(1)); + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(held, options), options); + Assert.That(back.Peek.Single().Id, Is.EqualTo(held.Peek.Single().Id), "the minted id of a private list survives a round trip"); + + Assert.That(Problems(typeof(HApPrivSet)), Is.Empty, "MessagePack reads an AllowPrivate type's private setter back"); + var privSet = new HApPrivSet { Items = { new EApPrivSet(), new EApPrivSet() } }; + Assert.That(Assign(privSet, "r", deterministic: false), Is.EqualTo(2)); + var round = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(privSet, options), options); + Assert.That(round.Items.Select(item => item.Id), Is.EqualTo(privSet.Items.Select(item => item.Id))); + Assert.That(round.Items.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public"), "without AllowPrivate it is still refused"); + } + + private static string?[] Ids(object holder) => holder switch + { + HInit init => init.Items.Select(item => item.Id).ToArray(), + HCtorId ctor => ctor.Items.Select(item => item.Id).ToArray(), + _ => throw new ArgumentException(holder.GetType().Name), + }; + + // A subclass that redeclares the id member does not change which member the declared element writes. + [Test] + public void AnElementIsReadByItsDeclaredTypeNotItsRuntimeType() + { + Assert.That(Problems(typeof(HDoc2)), Is.Empty); + static HE Hd(string? declared = null, string? redeclared = null) + { + var item = new HD { Id = redeclared }; + ((HE)item).Id = declared; + return item; + } + + var minted = new HDoc2 { Items = { Hd() } }; + Assert.That(Assign(minted, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(minted.Items[0].Id, Does.Match("^[0-9a-f]{12}$")); + Assert.That(((HD)minted.Items[0]).Id, Is.Null, "the redeclared member is never written, so it is not minted"); + + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "not-hex") } }, "r", false), + Throws.TypeOf(), "the declared member is checked"); + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "not-hex") } }, "r", false), + Throws.Nothing, "the redeclared member is never read"); + + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "0123456789ab") } }), Is.True); + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "0123456789ab") } }), Is.False); + } + // ---- shapes ---- + [MessagePackObject] public class PubIgn { [Key(0)] [IgnoreMember] public List L { get; set; } = new(); } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class ApIgn { [Key(0)] [IgnoreMember] private List l = new(); } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public class PgHold { [Key(0)] public List Items { private get; set; } = new(); } + [MessagePackObject] + public class MemberMix + { + [Key(0)] public int A; + [Key(1)] [IgnoreMember] public List? Ign; + [Key(2)] public string? PS { get; private set; } + [Key(3)] public string? GetOnly { get; } +#pragma warning disable CS0169, CS0414, CS0649 + [Key(4)] private int priv; +#pragma warning restore CS0169, CS0414, CS0649 + [Key(5)] public string? InitOnly { get; init; } + [IgnoreMember] public int Priv => priv; + } + + [MessagePackObject] public class EPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EIgnId { [Key(0)] [IgnoreMember] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HIgnId { [Key(0)] public List Items { get; set; } = new(); } + #pragma warning disable CS0169, CS0649 + [MessagePackObject] public class EPrivateId { [Key(0)] [CultElementId] private string? id; } +#pragma warning restore CS0169, CS0649 + [MessagePackObject] public class HPrivateId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EInit { [Key(0)] [CultElementId] public string? Id { get; init; } } + [MessagePackObject] public class HInit { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] + public class ECtorId + { + [SerializationConstructor] public ECtorId(string? id) { Id = id; } + [Key(0)] [CultElementId] public string? Id { get; set; } + } + [MessagePackObject] public class HCtorId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HE { [Key(0)] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HD : HE { [Key(2)] [CultElementId] public new string? Id { get; set; } } + [MessagePackObject] public class HDoc2 { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class KNode : IEnumerable + { + [Key(0)] [CultElementId] public string? Id { get; set; } + [Key(1)] public List Children { get; set; } = new(); + public IEnumerator GetEnumerator() => Children.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HKNode { [Key(0)] public List Nodes { get; set; } = new(); } + public class UNode : IEnumerable + { + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HUNode { [Key(0)] public UNode? N { get; set; } } + [MessagePackObject] + public class KBag : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + [Key(1)] public List Other { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] + public class KBagIded : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } [MessagePackObject] public class ENest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } [MessagePackObject] public class EBareNest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } + public sealed class NoFormatter { } + public class UnkeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [Union(0, typeof(KeyedArm)), Union(1, typeof(UnkeyedArm))] public abstract class ArmBase { } + [MessagePackObject] public class KeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HUnkeyedArmList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HUnkeyedArmField { [Key(0)] public ArmBase? Root { get; set; } } + [MessagePackObject] + public class HCollections + { + [Key(0)] public List A { get; set; } = new(); + [Key(1)] public EIded[] B { get; set; } = Array.Empty(); + [Key(2)] public Dictionary C { get; set; } = new(); + [Key(3)] public List> D { get; set; } = new(); + } + + [MessagePackObject] + public class HValueListsBesideObjects + { + [Key(0)] public List Objects { get; set; } = new(); + [Key(1)] public List Numbers { get; set; } = new(); + [Key(2)] public List Plain { get; set; } = new(); + [Key(3)] public List Points { get; set; } = new(); + } + + [MessagePackFormatter(typeof(NoFormatter))] public class Roles : List { } + [MessagePackObject] public class HRoles { [Key(0)] public Roles Items { get; set; } = new(); } + [MessagePackFormatter(typeof(NoFormatter))] public class PlainValues : List { } + [MessagePackObject] public class HFormattedValueCollection { [Key(0)] public List Objects { get; set; } = new(); [Key(1)] public PlainValues Plain { get; set; } = new(); } + + [MessagePackObject] + public abstract class FormattedIdBase + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] + public class FormattedIdOverride : FormattedIdBase + { + [Key(0)] [CultElementId] public override string Id { get; set; } = ""; + } + + [MessagePackObject] public class HOverFormattedId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class OwnerBaseFormatted + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual List Items { get; set; } = new(); + } + + [MessagePackObject] + public class OwnerOverFormatted : OwnerBaseFormatted + { + [Key(0)] public override List Items { get; set; } = new(); + } + + [MessagePackObject] + public class IdVirtual + { + [Key(0)] public string X { get; set; } = ""; + [Key(1)] [CultElementId] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] public class IdOverride : IdVirtual { [Key(1)] public override string Id { get; set; } = ""; } + [MessagePackObject] public class HIdOverride { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HUnkeyedTwoLists { [Key(0)] public List A { get; set; } = new(); [Key(1)] public List B { get; set; } = new(); } + public class PlainValue { public string X { get; set; } = ""; } + [MessagePackObject] public class HValueList { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedElement { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedElement { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedList { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public List Items { get; set; } = new(); } + [MessagePackObject] public class FormattedIdElement { [Key(0)] [CultElementId] [MessagePackFormatter(typeof(NoFormatter))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedOwner { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedOwner { [Key(0)] public FormattedOwner Owner { get; set; } = new(); } + [MessagePackObject] public class HFormattedMember { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public HHasIds Inner { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedLeaf { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class HFormattedField { [Key(0)] public FormattedLeaf Leaf { get; set; } = new(); [Key(1)] public List Tags { get; set; } = new(); } + + [MessagePackObject] public class FormattedDerived : FormattedElement { } + [MessagePackObject] public class HFormattedDerived { [Key(0)] public List Items { get; set; } = new(); } + [Union(0, typeof(IdArm))] public abstract class IdBase { } + [MessagePackObject] public class IdArm : IdBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Items { get; set; } = new(); } + [Union(0, typeof(PlainArm))] public abstract class PlainBase { } + [MessagePackObject] public class PlainArm : PlainBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HPlainBaseList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HIdBaseOwner { [Key(0)] public IdBase? Root { get; set; } } + + [MessagePackObject] public class HLeft { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HRight { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HCycle { [Key(0)] public HCycleBack Back { get; set; } = new(); } + [MessagePackObject] public class HCycleBack { [Key(0)] public HCycle? Up { get; set; } [Key(1)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HDiamond { [Key(0)] public HLeft Left { get; set; } = new(); [Key(1)] public HRight Right { get; set; } = new(); } + [MessagePackObject] public class HList { [Key(0)] public List Items { get; set; } = new(); } [MessagePackObject] public class HArray { [Key(0)] public EBare[] Items { get; set; } = Array.Empty(); } [MessagePackObject] public class HDictionary { [Key(0)] public Dictionary Map { get; set; } = new(); } @@ -337,6 +965,80 @@ namespace GameCult.Caching.Tests [Key(2)] public List Nest { get; set; } = new(); } + [MessagePackObject] public struct SHold { [Key(0)] public List Items; } + [MessagePackObject] public struct SHoldIded { [Key(0)] public List Items; } + [MessagePackObject] public struct SValue { [Key(0)] public int N; } + public struct UnkeyedS { [Key(0)] public List Items; } + [MessagePackObject] public class HUnkeyedStruct { [Key(0)] public UnkeyedS Value; } + public class InheritedBare : EBare { } + [MessagePackObject] public class HInheritedBare { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HNullableStructIded { [Key(0)] public SHoldIded? Hold; } + [MessagePackObject] public class HStruct { [Key(0)] public SHold Hold; } + [MessagePackObject] public class HStructList { [Key(0)] public List Holds { get; set; } = new(); } + [MessagePackObject] public class HNullableStruct { [Key(0)] public SHold? Hold; } + [MessagePackObject] public class HStructIded { [Key(0)] public SHoldIded Hold; [Key(1)] public List Many = new(); } + [MessagePackObject] public class HStructValues { [Key(0)] public List Structs = new(); [Key(1)] public List Objects = new(); } + + [MessagePackObject] public class HReadOnlyMap { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HReadOnlyIded { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HMapIded { [Key(0)] public Dictionary Map { get; set; } = new(); } + public sealed class RoMap : IReadOnlyDictionary + { + private readonly Dictionary _inner; + public RoMap(Dictionary inner) => _inner = inner; + public HHasIds this[string key] => _inner[key]; + public IEnumerable Keys => _inner.Keys; + public IEnumerable Values => _inner.Values; + public int Count => _inner.Count; + public bool ContainsKey(string key) => _inner.ContainsKey(key); + public bool TryGetValue(string key, out HHasIds value) => _inner.TryGetValue(key, out value!); + public IEnumerator> GetEnumerator() => _inner.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [MessagePackObject] public class BaseC { [Key(0)] public int X { get; set; } } + [MessagePackObject] public class DerC : BaseC { [Key(1)] public List L { get; set; } = new(); } + [MessagePackObject] public class HBase { [Key(0)] public BaseC B { get; set; } = new(); } + [MessagePackObject] public class HObject { [Key(0)] public object? O { get; set; } } + +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class HHiddenList { [Key(0)] private List Stash { get; set; } = new(); } + [MessagePackObject] public class HHiddenOwner { [Key(0)] private HHasIds stash = new(); } + [MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List stash = new(); [IgnoreMember] public List Peek => stash; } + [MessagePackObject] public class HHiddenBase { [Key(0)] private List stash = new(); } + [MessagePackObject] public class HHiddenFromBase : HHiddenBase { [Key(1)] public int N { get; set; } } + [MessagePackObject] public class HHiddenLeaf { [Key(0)] public List Items { get; set; } = new(); [Key(1)] private int secret = 7; } + [MessagePackObject] public class ApBase { [Key(0)] private List baseStash = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApDerived : ApBase { [Key(1)] private List own = new(); [Key(2)] public int N { get; set; } [Key(3)] public List Pg { private get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApBase2 { [Key(0)] private List baseStash = new(); } + [MessagePackObject] public class NoApDerived : ApBase2 { [Key(1)] public int N { get; set; } } +#pragma warning restore CS0169, CS0414, CS0649 + + [MessagePackObject] public class Whole { [Key(0)] public T? Value { get; set; } } + public class NkHold { public List Items = new(); } + [MessagePackObject] public class SkHold { [Key("items")] public List Items = new(); } + [MessagePackObject(keyAsPropertyName: true)] public class KpBase { [Key(0)] public List Items = new(); } + public class KpDerived : KpBase { } + [MessagePackObject] public class MpBase { [Key(0)] public List Items = new(); } + [System.Runtime.Serialization.DataContract] public class DcOverMp : MpBase { } + public class ApInheritor : ApDerived { } + [MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; } + [System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List Items = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIgnAp { [Key(0)] [IgnoreMember] private List l = new(); } + [MessagePackObject] public class PSkH { [Key(0)] public List Items = new(); [Key("x")] private int x = 1; } + [MessagePackObject] public class ISkH { [Key(0)] public List Items = new(); [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class SkOverBase { [Key("items")] public virtual List Items { get; set; } = new(); } + [MessagePackObject] public class SkOver : SkOverBase { public override List Items { get; set; } = new(); } + [MessagePackObject] public class NoIdEl { [Key(0)] public int N { get; set; } [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class NoIdPlain { [Key(0)] public int N { get; set; } } + [MessagePackObject(AllowPrivate = true)] public class EApIgnId { [Key(0)] public int N { get; set; } [Key(1)] [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HApIgnId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIdBase { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class ENoApPrivSet : ApIdBase { [Key(1)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HNoApPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HApPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; } [MessagePackObject] public class HDerivedText { [Key(0)] public List Items { get; set; } = new(); } diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index c5716975..bee0a5d5 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Reflection; @@ -356,6 +357,48 @@ namespace GameCult.Caching.Tests private static byte[] OldShapedReels(string label) => MessagePackSerializer.Serialize(new List { new() { Label = label, Marks = { new OldMark { Text = "x" }, new OldMark { Text = "y" } } } }); + // A holder the walk does not own (ImmutableArray here) is one whole value: its elements carry no element ids, so it never + // marks the store, and a variant replaces it whole (q3 amended 2026-10-16). + [Test] + public void AWholeListHolderNeverMarksTheStoreAndAVariantReplacesItWhole() + { + Assert.That(CultElementIds.CanHoldIds(typeof(IdWholeHolder)), Is.False); + var path = PathOf("whole-holder.cc"); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + cache.Commit(batch => batch.Upsert(typeof(IdWholeHolder), + new IdWholeHolder { Name = "base", Reels = ImmutableArray.Create(new IdReel { Label = "set", Id = HexA }, new IdReel { Label = "unset" }) }, + new CultRecordKey("base"))); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "ids inside a whole value mark nothing"); + + using (var fresh = OpenWith(path, false, typeof(IdWholeHolder))) + { + fresh.Commit(batch => batch.Upsert(typeof(IdWholeHolder), new IdWholeHolder { Name = "other" }, new CultRecordKey("other"))); + var back = fresh.Get(new CultRecordKey("base"))!; + Assert.That(back.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { HexA, "" }), "set ids are plain values, unset ids stay unset"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + var replacement = ImmutableArray.Create(new IdReel { Label = "v", Id = "eeeeeeeeeeee" }); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdWholeHolder.Name), "v"), cache.Override(nameof(IdWholeHolder.Reels), replacement) })); + Assert.That(cache.Get(new CultRecordKey("v"))!.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { "eeeeeeeeeeee" }), "replaced whole"); + Assert.That(cache.Get(new CultRecordKey("base"))!.Reels, Has.Length.EqualTo(2), "the base is untouched"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "a variant makes v2, and nothing in it holds element ids"); + using var again = OpenWith(path, false, typeof(IdWholeHolder)); + Assert.That(again.Get(new CultRecordKey("v"))!.Reels.Single().Id, Is.EqualTo("eeeeeeeeeeee")); + + var step = new CultVariantOverride(CultOverrideOp.Set, new[] { new CultPathStep(1, HexA) }, string.Empty, MessagePackSerializer.Serialize("x")); + var refused = Assert.Catch(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { again.Override(nameof(IdWholeHolder.Name), "p"), step })))!; + Assert.That(refused.ToString(), Does.Contain("top-level Set overrides only")); + } + [Test] public void AStoreThatCanHoldElementIdsCarriesTheV3MarkerAndOneThatCannotStaysV1() { @@ -1020,6 +1063,78 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1")); } + // Commits one plain item through a cache over `path` and reports how many copied records that write decoded to ask whether + // they hold an element id, and the header it left. + private (int Decoded, string Header) CommitItemCountingDecodes(string path, params Type[] types) + { + var registry = CultDocumentRegistry.ForTypes(types); + var store = new SingleFileMessagePackBackingStore(path); + using var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(store); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + store.CopiedRecordsDecoded = 0; + cache.Commit(batch => batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, new CultRecordKey("p"))); + return (store.CopiedRecordsDecoded, HeaderOf(path)); + } + + // The copy check decodes a record only where the answer can change: each gate, removed, makes one of these counts wrong. + [Test] + public void TheCopyCheckDecodesOnlyWhatCouldMarkTheStore() + { + // A v3 header is already marked: nothing to decide. + var marked = PathOf("gate-marked.cc"); + WritePreIdStore(marked, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(marked)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(marked, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(CommitItemCountingDecodes(marked, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v3")), "a marked store decodes nothing"); + + // Records whose type reaches no object list can hold no id. + var noLists = PathOf("gate-no-lists.cc"); + using (var seed = OpenWith(noLists, false, typeof(PreCut2FixtureItem))) + { + foreach (var key in new[] { "x", "y", "z" }) + UpsertPlain(seed, key); + } + + Assert.That(CommitItemCountingDecodes(noLists, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a type with no object list is not decoded"); + + // A written record that holds an id already marks the store, whichever of the written records it is. + var stagedPath = PathOf("gate-staged.cc"); + WritePreIdStore(stagedPath, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var stagedStore = new SingleFileMessagePackBackingStore(stagedPath); + var stagedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck), typeof(PreCut2FixtureItem) }); + using (var cache = new CultCache(stagedRegistry, CultCacheMessagePack.CreateCodec(stagedRegistry))) + { + cache.AddBackingStore(stagedStore); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + stagedStore.CopiedRecordsDecoded = 0; + cache.Commit(batch => + { + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh")); + }); + } + + Assert.That(stagedStore.CopiedRecordsDecoded, Is.EqualTo(0), "a staged record holding an id marks without asking"); + Assert.That(HeaderOf(stagedPath), Is.EqualTo("cultcache.store.v3")); + + // Id-capable records without ids: each is decoded once, and the store stays unmarked. + var capable = PathOf("gate-capable.cc"); + WritePreIdStore(capable, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(capable, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((3, "cultcache.store.v1"))); + + // A record the writer's registry does not claim is foreign: no type, no decode. + var foreign = PathOf("gate-foreign.cc"); + WritePreIdStore(foreign, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(foreign, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a foreign record is not decoded"); + + // A variant is copied as its bytes: only a C# write of one holding ids marks the store, and that write marked it already. + var variant = PathOf("gate-variant.cc"); + WriteV2StoreWithAnIdLessVariant(variant); + Assert.That(CommitItemCountingDecodes(variant, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((1, "cultcache.store.v2")), "the plain base is decoded, the variant is not"); + } + // Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta // it read, so the file holds none and no write of a plain record beside it may mark the file. private static void WriteV2StoreWithAnIdLessVariant(string path) @@ -1215,6 +1330,14 @@ namespace GameCult.Caching.Tests [Key(1)] public int Depth { get; set; } } + [CultDocument("tests.element_id_whole_holder", "tests.element_id_whole_holder.v1")] + [MessagePackObject] + public sealed class IdWholeHolder + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public ImmutableArray Reels { get; set; } = ImmutableArray.Empty; + } + [CultDocument("tests.element_id_deck", "tests.element_id_deck.v1")] [MessagePackObject] public sealed class IdDeck diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index cc76c760..f83ae98d 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -67,6 +67,181 @@ namespace GameCult.Caching.Tests } } + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledAsync(string path) + { + var cache = new CultCache(); + var store = new SingleFileMessagePackBackingStore(path); + cache.AddBackingStore(store); + await cache.PullAllBackingStoresAsync(); + return (cache, store); + } + + // cultcache-py writes element ids under an unmarked v1 header. A C# write that carries such a record forward marks the + // store, because a reader from before ids would otherwise rewrite the elements without them. + [Test] + public async Task ACSharpWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "cultcache-py leaves the header unmarked"); + var (cache, store) = await PulledAsync(path); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // What a load saw is a fact about those bytes at that version. Another writer replaced the record since, so the copy this + // write makes is not the record the load saw, and the header is not judged by it. + [Test] + public async Task ACopiedRecordAnotherWriterChangedIsNotJudgedByTheOldRead() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + var (reader, readerStore) = await PulledAsync(path); + + // Another writer replaces the record with one that holds no id. + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var replaced = snapshot.Records.Single(record => record.Key == "py"); + replaced.Payload = CultDocumentMessagePackSerialization.SerializeUntyped( + new NumericLeafVectorTests.LeafRecord { Id = "py" }, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the replacement holds no id"); + + await reader.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + readerStore.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // A v1 store holding only this record c0, opened by a cache that has pulled it; then another runtime appends the python + // record (and its payload rewritten by `change`) to the file raw, header still v1, the way a writer that keeps the v1 it + // found does. Returns the open cache and its store. + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledThenAppended( + string path, string pySource, Action? change = null) + { + using (var seed = new CultCache()) + { + var seedStore = new SingleFileMessagePackBackingStore(path); + seed.AddBackingStore(seedStore); + await seed.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c0" }); + seedStore.PushAll(); + } + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var opened = await PulledAsync(path); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var pySnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource)); + var py = pySnapshot.Records.Single(record => record.Key == "py"); + change?.Invoke(py); + snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray(); + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Concat(pySnapshot.SchemaCatalog).GroupBy(entry => entry.SchemaId).Select(group => group.First()).ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found"); + return opened; + } + + // The cache never read this record: it was appended after the pull. A per-instance memory of what the pull saw cannot + // know about it; the write decides from the bytes the file holds under its lock. + [Test] + public async Task ARecordAForeignWriterAddsAfterThePullMarksTheStoreWhenCopied() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "after-pull.cc"); + var (cache, store) = await PulledThenAppended(path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc"))); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // Ids a load minted in memory are not on disk: the check judges the file's bytes, never the document the cache holds. + [Test] + public async Task ARecordWhoseIdsExistOnlyInMemoryLeavesTheStoreUnmarked() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "in-memory.cc"); + var source = Store("leaf-record.cc", scratch); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(source)); + var idless = new NumericLeafVectorTests.LeafRecord { Id = "py" }; + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "one" }); + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "two" }); + snapshot.Records.Single().Payload = CultDocumentMessagePackSerialization.SerializeUntyped(idless, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single().Payload; + + var (cache, store) = await PulledAsync(path); + Assert.That(cache.AllEntries.OfType().Single().Items.Select(item => item.Id), Has.All.Length.EqualTo(12), "the load minted ids in memory"); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py").Payload; + Assert.That(after, Is.EqualTo(before), "the copied record is the bytes the file held"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // The write-set rule copies records without interpreting them, so the check must never refuse a write: a record this + // cache claims but cannot decode is copied as its bytes and decides nothing. + [Test] + public async Task ACopiedRecordThatDoesNotDecodeIsCopiedAndDecidesNothing() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "undecodable.cc"); + var garbage = new byte[] { 0xc1, 0xc1, 0xc1 }; + var (cache, store) = await PulledThenAppended( + path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc")), record => record.Payload = garbage); + + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + Assert.DoesNotThrow(() => store.PushAll()); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py"); + Assert.That(py.Payload, Is.EqualTo(garbage)); + } + finally + { + Directory.Delete(scratch, true); + } + } + [TestCase(3)] [TestCase(5)] [TestCase(7)] diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index a727ba75..85d17832 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.Serialization; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; @@ -20,7 +21,7 @@ namespace GameCult.Caching.Tests { private const string Canary = "CANARY-9d1e47"; - private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc) }); + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc), typeof(StubDoc) }); private delegate void Emit(ref MessagePackWriter writer); @@ -246,15 +247,543 @@ namespace GameCult.Caching.Tests [Test] public void ConsumerResolverStillEncodesArms() { - var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new GuardDoc { Id = "d1", Root = new StubArm() }, typeof(GuardDoc), Registry); - Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Doc((ref MessagePackWriter w) => + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new StubDoc { Id = "d1", Root = new StubArm() }, typeof(StubDoc), Registry); + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Raw((ref MessagePackWriter w) => { + w.WriteArrayHeader(2); w.Write("d1"); w.WriteArrayHeader(2); w.Write(9); w.WriteArrayHeader(1); w.Write("via-resolver"); })))); - var back = (GuardDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(GuardDoc), bytes, Registry); + var back = (StubDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(StubDoc), bytes, Registry); Assert.That(back.Root, Is.TypeOf()); } + // A list element is written by its [Key] layout or its id never reaches disk: a consumer resolver that claims the arm of + // a list's union is refused on every path that encodes or decodes the document, naming the arm and the resolver, and + // writes nothing. + [Test] + public void ListElementsClaimedByAResolverAreRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ClaimDoc) }); + var doc = new ClaimDoc { Id = "c1", Items = { new ClaimArm { N = 4 } } }; + void Refused(TestDelegate attempt) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain("ClaimArm").And.Contain(nameof(ClaimArmResolver)).And.Contain(nameof(ClaimDoc))); + } + + Refused(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ClaimDoc), registry)); + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("c1"); w.WriteArrayHeader(0); }); + Refused(() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ClaimDoc), held, registry)); + Refused(() => CultCacheMessagePack.CreateCodec(registry).Serialize(doc.Items, typeof(List), typeof(ClaimDoc))); + + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "claim.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + Assert.That(Assert.Catch(() => cache.Commit(batch => batch.Upsert(typeof(ClaimDoc), doc, new CultRecordKey("c1"))))!.ToString(), + Does.Contain(nameof(ClaimArmResolver))); + Assert.That(File.Exists(path), Is.False, "nothing reached disk"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // A resolver's formatter for the list type of an object list drops the elements' ids as surely as one for the element, + // so the collection type is on the id path (Soul s1 P1). A claim on a value list, even beside an object list, is none. + [Test] + public void AResolverClaimingTheListTypeOfAnObjectListIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(LcDoc), typeof(LcValueDoc) }); + var doc = new LcDoc { Id = "l1", Items = { new LcArm { Id = "000000000001", N = 4 } } }; + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("l1"); w.WriteArrayHeader(0); }); + foreach (var attempt in new TestDelegate[] + { + () => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(LcDoc), registry), + () => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcDoc), held, registry) + }) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain("List`1").And.Contain(nameof(LcDoc))); + } + + var values = new LcValueDoc + { + Id = "v1", + Items = { new LcOther { Id = "000000000002" } }, + Values = { new LcValue { N = 3 }, new LcValue { N = 5 } }, + Points = { new Pair { A = 1, B = 2 } } + }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(values, typeof(LcValueDoc), registry); + var back = (LcValueDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcValueDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000002")); + Assert.That(back.Values.Select(value => value.N), Is.EqualTo(new[] { 3, 5 })); + Assert.That(back.Points.Single(), Is.EqualTo(new Pair { A = 1, B = 2 })); + } + + // A resolver that hands out MessagePack's own formatter for a type writes its [Key] layout: it claims nothing, however it + // got there, and the document encodes with the id present (Soul s1 P2). + [Test] + public void AResolverHandingBackTheStandardFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(FbDoc) }); + var doc = new FbDoc { Id = "f1", Items = { new FbArm { Id = "000000000009", N = 7 } } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(FbDoc), registry); + + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(doc, CultDocumentMessagePackSerialization.Options)))); + var back = (FbDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(FbDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000009")); + } + + // A consumer whose resolver composite ends in StandardResolverAllowPrivate (some models keep private [Key] members) gets + // that resolver's formatter for every keyed type. It writes the [Key] layout plus the private members, so it is no + // claim: the document encodes, the id is on disk, and the private value survives. + [Test] + public void AStandardResolverAllowPrivateFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApDoc), typeof(ApPlainDoc) }); + var doc = new ApDoc { Id = "a1", Items = { new ApArm { Id = "00000000000a", N = 4 }.WithSecret(11) } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApDoc), registry); + + var back = (ApDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ApDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("00000000000a")); + Assert.That(back.Items.Single().Secret, Is.EqualTo(11), "a private plain-value [Key] member is written"); + + var plain = new ApPlainDoc { Id = "a2", Items = { new ApPlainArm { Id = "00000000000b", N = 5 } } }; + var plainBytes = CultDocumentMessagePackSerialization.SerializeUntyped(plain, typeof(ApPlainDoc), registry); + Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); + } + + // The walk reads a [MessagePackObject] type by its keys, but MessagePack writes one that is also an ICollection as a + // collection and drops the keys. The codec compares the two for every type on the id path and refuses the document. + [Test] + public void AKeyedTypeMessagePackWritesAsACollectionIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(KCollDoc), typeof(KBagDoc) }); + Assert.That(CultElementIds.Inspect(typeof(KCollDoc)).IdPath, Does.Contain(typeof(KColl))); + + var coll = new KCollDoc { Id = "k1" }; + coll.Items.Add(new FbArm { Id = "00000000000c" }); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(coll, typeof(KCollDoc), registry))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KColl)} as a collection, but the element-id walk reads it by its [Key] layout")); + + // A keyed map is read by key too, so a keyed Dictionary subclass is on the path and refused the same way. + var map = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(KDictDoc)))!; + Assert.That(map.Message, Does.Contain($"writes {nameof(KDict)} as a collection")); + + var bag = new KBagDoc { Id = "b1", Bag = new KBagE { Items = { new FbArm { Id = "00000000000d" } } } }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(bag, typeof(KBagDoc), registry); + var back = (KBagDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(KBagDoc), bytes, registry); + Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d")); + } + + // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. + [Test] + public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter() + { + foreach (var holder in new[] + { + typeof(G>), typeof(G), typeof(G), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G), typeof(G), typeof(G), + }) + { + if (holder != typeof(G)) + Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.ToString()); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + } + + // One recogniser decides what the walk owns and the door only verifies it: the same types give no registration problem and a + // passing door. A type formatted by attribute or written by the object resolver is one whole value, off the id path. + [Test] + public void OneRecogniserDecidesAndTheDoorAgrees() + { + foreach (var holder in new[] { typeof(G), typeof(G), typeof(G), typeof(G), typeof(G) }) + { + Assert.That(CultElementIds.Inspect(holder).Problems, Is.Empty, holder.ToString()); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + + foreach (var whole in new[] { typeof(G), typeof(G), typeof(G), typeof(G) }) + Assert.That(CultElementIds.Inspect(whole).IdPath, Is.Empty, whole.ToString()); + var ap = CultElementIds.Inspect(typeof(G)).IdPath; + Assert.That(ap, Does.Contain(typeof(ApTrueHold)).And.Contain(typeof(List))); + var seq = new G { Value = new DcSeq { Items = { new FbArm() } } }; + Assert.That(CultElementIds.Plan(seq.GetType(), seq, "r", deterministic: false).Count, Is.EqualTo(0)); + } + + // MessagePack throws for a type, or StandardResolver has no formatter for it (an interface or abstract class that is only + // IEnumerable): such a type cannot be persisted, so the door refuses it with the same typed text as a throwing build. + [Test] + public void ATypeMessagePackCannotBuildIsATypedRefusal() + { + foreach (var (holder, name) in new[] { (typeof(G), nameof(IArms)), (typeof(G), nameof(AbsArms)) }) + { + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString())!; + Assert.That(thrown.Message, Does.Contain(name).And.Contain("cannot build a formatter")); + Assert.That(thrown.Message, Does.Contain("StandardResolver has no formatter for it"), "a null formatter says so, without a thrown message"); + } + } + + // A keyed type that enumerates lists is read by key and is on the id path, so the door checks it: MessagePack writes it as a + // collection, which loses the ids the walk would mint. + [Test] + public void AKeyedTypeThatEnumeratesListsIsCheckedByTheDoor() + { + Assert.That(CultElementIds.Inspect(typeof(G)).IdPath, Does.Contain(typeof(KLL))); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KLL)} as a collection")); + } + + // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter + // the claim check asks about (the claim stands) or a resolver ahead (it fails to build). + [Test] + public void AResolverThatThrowsIsATypedRefusal() + { + var first = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc)))!; + Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl))); + var second = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!; + Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build").And.Not.Contain("target of an invocation"), + "the resolver's own exception is named, not the reflection wrapper"); + + // A type MessagePack itself cannot build a formatter for (two public members on one key) is refused, naming the type. + var third = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc3)))!; + Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter").And.Not.Contain("target of an invocation")); + } + + [MessagePackObject] + public sealed class KColl : List + { + [Key(0)] public int N; + } + + [CultDocument("tests.kcoll_doc", "tests.kcoll_doc.v1")] + [MessagePackObject] + public sealed class KCollDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KColl Items = new(); + } + + [MessagePackObject] + public sealed class KDict : Dictionary + { + [Key(0)] public int N; + } + + [CultDocument("tests.kdict_doc", "tests.kdict_doc.v1")] + [MessagePackObject] + public sealed class KDictDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KDict Items = new(); + } + + [MessagePackObject] + public sealed class KBagE : IEnumerable + { + [Key(0)] public List Items = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [CultDocument("tests.kbag_doc", "tests.kbag_doc.v1")] + [MessagePackObject] + public sealed class KBagDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KBagE Bag = new(); + } + + [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } + [Union(0, typeof(GuArm))] public interface IGu { } + [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [MessagePackObject] public sealed class GHold { [Key(0)] public List Items = new(); } + [MessagePackObject] public struct GStruct { [Key(0)] public List Items; } + +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } + [MessagePackObject] public sealed class DkEl2 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl3 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] public int M; } + [CultDocument("tests.dk_doc3", "tests.dk_doc3.v1")] + [MessagePackObject] + public sealed class DkDoc3 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [CultDocument("tests.dk_doc", "tests.dk_doc.v1")] + [MessagePackObject] + public sealed class DkDoc { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [CultDocument("tests.dk_doc2", "tests.dk_doc2.v1")] + [MessagePackObject] + public sealed class DkDoc2 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + + [MessagePackFormatter(typeof(FormattedRolesFormatter))] public sealed class FormattedRoles : List { } + public sealed class FormattedRolesFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FormattedRoles value, MessagePackSerializerOptions options) => writer.WriteNil(); + public FormattedRoles Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new FormattedRoles(); } + } + + [DataContract] public sealed class DcSeq : IEnumerable + { + [DataMember(Order = 0)] public List Items = new(); + public IEnumerator GetEnumerator() => Items.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + public interface IArms : IEnumerable { } + public abstract class AbsArms : IEnumerable + { + public abstract IEnumerator GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public sealed class KLL : List> { [Key(0)] public int N; } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject(AllowPrivate = true)] public sealed class ApTrueHold { [Key(0)] private List items = new(); } +#pragma warning restore CS0169, CS0414, CS0649 + [DataContract] public sealed class DcColl : List { [DataMember(Order = 0)] public int N; } + [DataContract] public sealed class DcBad : List { [DataMember(Order = 0)] public int A; [DataMember(Order = 0)] public int B; } + + public sealed class DkFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N); + public DkEl Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => new DkEl { N = reader.ReadInt32() }; + } + + [MessagePackObject] + public sealed class ApArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + [Key(2)] private int secret; + [IgnoreMember] public int Secret => secret; + public ApArm WithSecret(int value) + { + secret = value; + return this; + } + } + + [CultDocument("tests.ap_doc", "tests.ap_doc.v1")] + [MessagePackObject] + public sealed class ApDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [MessagePackObject] + public sealed class ApPlainArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.ap_plain_doc", "tests.ap_plain_doc.v1")] + [MessagePackObject] + public sealed class ApPlainDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [MessagePackObject] + public sealed class LcArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [MessagePackObject] + public sealed class LcOther + { + [Key(0)] [CultElementId] public string Id = string.Empty; + } + + public sealed class LcValue + { + public int N; + } + + [CultDocument("tests.lc_doc", "tests.lc_doc.v1")] + [MessagePackObject] + public sealed class LcDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [CultDocument("tests.lc_value_doc", "tests.lc_value_doc.v1")] + [MessagePackObject] + public sealed class LcValueDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + [Key(2)] public List Values = new(); + [Key(3)] public List Points = new(); + } + + [MessagePackObject] + public sealed class FbArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.fb_doc", "tests.fb_doc.v1")] + [MessagePackObject] + public sealed class FbDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + // Claims List (an object list's type: refused), List and List (value lists: accepted, as a math + // resolver claims List), and answers FbArm with MessagePack's own formatter (no claim). + public sealed class ListClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ListClaimResolver(); + + public IMessagePackFormatter? GetFormatter() + { + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new NilFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); + if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(DkEl)) return (IMessagePackFormatter)(object)new DkFormatter(); + if (typeof(T) == typeof(DkEl2)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); + if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || + typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) + return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); + return null; + } + + private sealed class NilFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new List(); + } + } + + private sealed class ValuesFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) writer.Write(item.N); + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) list.Add(new LcValue { N = reader.ReadInt32() }); + return list; + } + } + + private sealed class PointsFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) { writer.WriteArrayHeader(2); writer.Write(item.A); writer.Write(item.B); } + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) + { + reader.ReadArrayHeader(); + list.Add(new Pair { A = reader.ReadSingle(), B = reader.ReadSingle() }); + } + + return list; + } + } + } + + // Composed after ListClaimResolver: once that resolver has answered FbArm with the standard formatter the composite never + // asks this one, so its claim on the same type must not refuse the document. + public sealed class FbLaterClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new FbLaterClaimResolver(); + + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(FbArm) ? (IMessagePackFormatter)(object)new LaterFormatter() : null; + + private sealed class LaterFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FbArm value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public FbArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new FbArm(); + } + } + } + + [Test] + public void OptionsRefuseANullDocumentTypeOrAssembly() + { + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsFor(null!)); + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(null!)); + } + + [Union(0, typeof(ClaimArm))] + public abstract class ClaimNode { } + + [MessagePackObject] + public sealed class ClaimArm : ClaimNode + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.claim_doc", "tests.claim_doc.v1")] + [MessagePackObject] + public sealed class ClaimDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + public sealed class ClaimArmResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ClaimArmResolver(); + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(ClaimArm) ? (IMessagePackFormatter)(object)new ClaimFormatter() : null; + + private sealed class ClaimFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, ClaimArm value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(1); + writer.Write(value.N); + } + + public ClaimArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.ReadArrayHeader(); + return new ClaimArm { N = reader.ReadInt32() }; + } + } + } + // The guard wraps the composite, so the composite's own order is still the contract: a consumer resolver outranks the // document resolver (CultRecordRef) and MessagePack's own object formatter, here for a type it also knows. [Test] @@ -568,7 +1097,7 @@ namespace GameCult.Caching.Tests } // An arm is [key, armSlots]; the keys here start at 3, skip 0-2, 4, 6 and 7, and never reuse a retired key. - [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(StubArm))] + [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(Nine))] public abstract class GuardNode { } [MessagePackObject] @@ -592,11 +1121,25 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class Unit8 : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Nine : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + // A subtype the union does not name. [MessagePackObject] public sealed class Rogue : GuardNode { [Key(0)] public string Note = string.Empty; } - // Written and read by a consumer resolver, not by MessagePack. - public sealed class StubArm : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + // Written and read by a consumer resolver, not by MessagePack. It can only sit outside a list: a list element is written + // by its [Key] layout, or its id never reaches disk (ListElementsClaimedByAResolverAreRefused). + [Union(9, typeof(StubArm))] + public abstract class StubNode { } + + public sealed class StubArm : StubNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + + [CultDocument("tests.stub_doc", "tests.stub_doc.v1")] + [MessagePackObject] + public sealed class StubDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public StubNode? Root; + } public sealed class StubArmResolver : IFormatterResolver { diff --git a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs index b5f63775..a58bc29d 100644 --- a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs +++ b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs @@ -5,6 +5,7 @@ using GameCult.Caching.MessagePack; using GameCult.Mesh; using GameCult.Networking; using MessagePack; +using MessagePack.Resolvers; using NUnit.Framework; using R3; using System; @@ -193,6 +194,19 @@ namespace GameCult.Geometry.Tests .Should().Be("geometry:domain:02c9b5810977406b0c206f3a3494327a423abb9448192be1b9a1863cd0f2ed95"); } + // A source-generated formatter is of no known family and passes the element-id door (ruling c2a-source-generated-formatters-defaulted). + [Test] + public void AGeneratedFormatterPassesTheElementIdDoor() + { + StandardResolver.Instance.GetFormatter().Should() + .NotBeSameAs(DynamicObjectResolver.Instance.GetFormatter(), "the fixture must stay source-generated"); + var payload = File.ReadAllBytes(FixturePath("ragnarok-domain.msgpack")); + var domain = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultGeometryDomainDocument), payload); + var again = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped( + typeof(CultGeometryDomainDocument), CultDocumentMessagePackSerialization.SerializeUntyped(domain, typeof(CultGeometryDomainDocument))); + CultGeometryDomainDocument.CreateRecordKey(again).Value.Should().Be(CultGeometryDomainDocument.CreateRecordKey(domain).Value); + } + [Test] public void RustExportedChunkArtifact_DecodesWithStableCultCacheKey() { diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 0ccf3f17..a8c501de 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using FluentAssertions; @@ -171,4 +172,129 @@ public sealed class CultMeshSingleFileElementIdTests }); cache.Get(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first); } + + // A list document defined in this assembly: the record another writer leaves holding ids under an unmarked header. + [CultDocument("mesh.test.id_list", "mesh.test.id_list.v1")] + [MessagePackObject] + public sealed class MeshIdList + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Items { get; set; } = new(); + } + + [MessagePackObject] + public sealed class MeshIdItem + { + [Key(0)] public string N { get; set; } = ""; + [Key(1)] [CultElementId] public string Id { get; set; } = ""; + } + + private static readonly CultRecordKey IdsKey = new("ids"); + private const string HexB = "bbbbbbbbbbbb"; + + // A v1 file whose record holds 12-hex element ids: the document is written with the type (v3), then the header is put back + // to the v1 a writer that does not decide would have left. + private string UnmarkedStoreHoldingIds() + { + var path = Path_("unmarked.cc"); + CultMesh.WriteSingleFileDocument( + path, + IdsKey, + new MeshIdList { Name = "ids", Items = { new MeshIdItem { Id = HexA }, new MeshIdItem { Id = HexB } } }, + registry: Registry(typeof(MeshIdList))); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + HeaderOf(path).Should().Be("cultcache.store.v1"); + return path; + } + + private string PythonStore() + { + string? vectors = null; + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null && vectors == null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "numeric-leaves", "py-store", "leaf-record.cc"); + if (File.Exists(candidate)) vectors = candidate; + } + + vectors.Should().NotBeNull("the shared vectors sit above the test directory"); + var path = Path_("leaf-record.cc"); + File.Copy(vectors!, path); + HeaderOf(path).Should().Be("cultcache.store.v1", "cultcache-py leaves the header unmarked"); + return path; + } + + private static CultDocumentRegistry Registry(params Type[] types) => CultDocumentRegistry.ForTypes(types); + + // A CultMesh write is a writer with a type: it decides for the copied record from the bytes the file holds, whatever any + // earlier read saw (a fresh store per call has read nothing). + [Test] + public void ACultMeshWriteThatCopiesARecordHoldingIdsMarksTheStore() + { + var path = UnmarkedStoreHoldingIds(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.ReadSingleFileDocument(path, IdsKey, Registry(typeof(MeshIdList))).Items.Select(item => item.Id).Should().Equal(HexA, HexB); + } + + // A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read. + [Test] + public void ARawCultMeshWriteBesideARecordHoldingIdsLeavesTheHeaderAlone() + { + var path = UnmarkedStoreHoldingIds(); + + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 }); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // Soul s1 P8, committed: the writer's registry has only the type it writes, so the python record is foreign to it and it + // decides nothing for it (variants:question:non-csharp-writers-mark-ids-2). + [Test] + public void ACultMeshWriteWhoseRegistryLacksTheRecordsTypeDecidesNothingForIt() + { + var path = PythonStore(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // The copy check never refuses a write: a record the registry claims by schema name but cannot resolve (the catalog entry + // names another type for a slot) is copied as its bytes and decides nothing. Soul s2 P5, incompatible-catalog-slot12. + [Test] + public void AnUnresolvableCopiedRecordIsCopiedAndDecidesNothing() + { + var path = UnmarkedStoreHoldingIds(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + const string changedId = "mesh.test.id_list.v0"; + var entry = snapshot.SchemaCatalog.Single(candidate => candidate.SchemaName == "mesh.test.id_list"); + entry.SchemaId = changedId; + entry.ContentHash = "persisted-by-another-runtime"; + entry.Members.Single(member => member.Slot == 1).TypeName = "System.Collections.Generic.List`1[[Other.Inner]]"; + snapshot.Records.Single(record => record.Key == IdsKey.Value).SchemaId = changedId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + + Assert.DoesNotThrow(() => CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument)))); + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + after.SchemaId.Should().Be(changedId); + after.Payload.Should().Equal(before.Payload); + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // Typed writes beside records that hold no id leave the store unmarked: the check marks only on an id it found. + [Test] + public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked() + { + var path = Path_("plain.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + CultMesh.WriteSingleFileDocument(path, new CultRecordKey("second"), Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } }