GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge element ids parts 1 and 2 into C2a

Browse source 
hands/variants-c2a-element-id-holes at 64ec14f9 (cuts c2a-element-id-holes r6
and c2a-id-closed-world r2, held by Soul s5/s6 and closed-world s1/s2). C2a's
side since the fork is two prose-map commits; no code conflict.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-16 04:10:01 +02:00
commit a5c7954eed
16 changed files with 2280 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs
View file

@ -23,10 +23,10 @@ namespace GameCult.Caching.MessagePack
// options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type
// deserializes as a registered document, a member type as itself.
public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec(
(value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)),
(value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsForDocument(document)),
(type, document, bytes) => type == document
? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry)
: global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)),
: global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsForDocument(document)),
CultDocumentMessagePackSerialization.OverlaySlots);
// The inspection model over that codec.

131
src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
View file

@ -38,11 +38,23 @@ public static class CultDocumentMessagePackSerialization
private const int SchemaCatalogMemberFieldCount = 8;
private const int StoreSnapshotFieldCount = 3;
public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty<IFormatterResolver>());
private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty<IFormatterResolver>());
private static readonly ConcurrentDictionary<Assembly, MessagePackSerializerOptions> AssemblyOptions = new();
public static readonly MessagePackSerializerOptions Options = Plain.Options;
public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly)
// Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of
// StandardResolver, the ones that can claim a type before its [Key] layout does.
private static readonly ConcurrentDictionary<Assembly, (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead)> AssemblyOptions = new();
// Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers
// are fixed for the process, so this is a cache of a pure answer.
private static readonly ConcurrentDictionary<Type, string?> DocumentProblems = new();
private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!;
public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options;
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly)
{
if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly));
return AssemblyOptions.GetOrAdd(documentAssembly, static assembly =>
@ -50,19 +62,118 @@ public static class CultDocumentMessagePackSerialization
var resolvers = assembly.GetCustomAttributes<CultCacheFormatterResolverAttribute>()
.Select(attribute => CreateResolver(attribute.ResolverType))
.ToArray();
return resolvers.Length == 0 ? Options : Compose(resolvers);
return resolvers.Length == 0 ? Plain : Compose(resolvers);
});
}
private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers)
// The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key]
// layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them
// without the id: refused here, once per document type, on every encode and decode path.
internal static MessagePackSerializerOptions OptionsForDocument(Type documentType)
{
if (documentType == null) throw new ArgumentNullException(nameof(documentType));
var (options, ahead) = ComposedFor(documentType.Assembly);
var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead));
return problem == null ? options : throw new InvalidOperationException(problem);
}
private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead)
{
foreach (var type in CultElementIds.Inspect(documentType).IdPath)
{
var generic = GetFormatterMethod.MakeGenericMethod(type);
// A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the
// composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout
// plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing
// either.
object? standard;
try
{
standard = generic.Invoke(StandardResolver.Instance, null);
}
catch (Exception exception)
{
return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message);
}
if (standard == null)
return CannotBuild(documentType, type, "StandardResolver has no formatter for it");
if (WrittenAsCollection(type, generic, standard))
return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection<T>, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";
var allowPrivateAsked = false;
object? allowPrivate = null;
foreach (var resolver in ahead)
{
object? formatter;
try
{
formatter = generic.Invoke(resolver, null);
}
catch (Exception exception)
{
return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type.";
}
if (formatter == null)
continue;
if (ReferenceEquals(formatter, standard))
break;
if (!allowPrivateAsked)
{
allowPrivateAsked = true;
try
{
allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null);
}
catch (Exception)
{
allowPrivate = null;
}
}
if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate))
break;
return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";
}
}
return null;
}
private static string CannotBuild(Type documentType, Type type, string reason) =>
$"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason}).";
// The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key
// (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed
// ICollection<T> or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework
// types, so neither can contradict. Any other formatter, a source-generated one among them, passes.
private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) =>
CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard);
private static object? Formatter(IFormatterResolver resolver, MethodInfo generic)
{
try
{
return generic.Invoke(resolver, null);
}
catch (Exception)
{
return null;
}
}
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
{
// The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer
// resolver still encodes an arm while the key check runs.
var inner = CompositeResolver.Create(
consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).Append(StandardResolver.Instance).ToArray());
return MessagePackSerializerOptions.Standard
var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
var options = MessagePackSerializerOptions.Standard
.WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner))
.WithSecurity(CultMessagePackSecurity.Instance);
return (options, ahead);
}
private static IFormatterResolver CreateResolver(Type resolverType)
@ -84,7 +195,7 @@ public static class CultDocumentMessagePackSerialization
{
if (registry == null) throw new ArgumentNullException(nameof(registry));
if (value != null) registry.GetRequired(type);
var payload = MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly));
var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type));
CultPayloadDepth.Check(payload, type, PayloadDirection.Write);
return payload;
}
@ -99,7 +210,7 @@ public static class CultDocumentMessagePackSerialization
if (registry == null) throw new ArgumentNullException(nameof(registry));
registry.GetRequired(type);
CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly))
return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
}

1
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -363,6 +363,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
currentManifest.FormatVersion,
holdsIds.Values.Select(holds => (bool?)holds).ToArray(),
copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)),
copiedHoldIds: false, // only C# writes a directory store, and it marks by content
directoryStore: true,
holdsVariants: false);
WriteManifest(targetCatalog, catalogBytes, currentIndex.Values

52
src/GameCult.Caching/Contracts/cultcache-persistence-format.md
View file

@ -336,8 +336,11 @@ an element id, variants or not). A single-file store declares `v3` when any reco
holds carries an element id: a non-empty `[CultElementId]` value in a record's document,
or in a variant's override values. A store that cannot read a variant's override values
(no codec attached) counts the variant as holding one. An element id is an ordinary keyed
string in the payload, so no reader can tell from a record's bytes whether it holds one;
a writer knows it only for the records it encodes. A write that copies any record forward
string in the payload, so only a reader with the record's type can tell; a C# write
decides for the records it encodes and, while the header it found is unmarked, for every
plain record it copies forward that its registry can read, decoding the bytes the file
holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload,
decide nothing. A write that copies any record forward
therefore keeps a `v3` header it found. Only a write that encodes every record the store
will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds
an id. A directory store's manifest says
@ -405,11 +408,18 @@ fifth slot:
## Element Ids
Every object that sits in a list, array or dictionary value of a registered document
type, union subtypes included, carries an id: one string member with an integer `[Key]`
and `[CultElementId]`. The id identifies the element inside its list, so an override can
name it (a path step is `[slot, elementId]`) and a base edit that reorders or inserts
elements retargets nothing.
Every object that sits in a list, array or dictionary value of a shape the element-id walk
owns carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The
owned shapes are arrays, `IEnumerable<T>` classes and interfaces, `IDictionary<,>` and
`IReadOnlyDictionary<,>` values, and `[MessagePackObject]` or `[Union]` types with integer
keys (a keyed struct included), read through the members MessagePack writes. Union subtypes
are included. The id identifies the element inside its list, so an override can name it (a
path step is `[slot, elementId]`) and a base edit that reorders or inserts elements
retargets nothing.
Any other shape is one whole value. Its elements carry no element ids: a `[CultElementId]`
member inside it is a plain string, never minted or checked and never counted for `v3`. A
variant replaces it whole (operator, 2026-10-16).
- A random id is 12 lowercase hex characters. A derived id
(`[CultElementId(nameof(Source))]`, for content-addressed elements) is the
@ -423,8 +433,32 @@ elements retargets nothing.
the record's next write, or `CultCache.MintElementIds()`.
- A store that holds an id declares the `v3` header (above), so a reader older than ids
refuses it instead of skipping the id slot and rewriting the elements without ids.
TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they
rewrite a store; they do not decide when a document carries ids.
A C# write also marks the store when a record it copies forward holds an id, judged
from the file's bytes at the write (above). TypeScript,
Python and Rust read `v3` and keep a `v3` header they found when they rewrite a
store; they carry ids and do not decide when a document carries them.
- An object list's elements are written by their `[Key]` layout, because the id is a
keyed member of the element. Value lists are replaced whole: a list of a non-keyed
class or struct, or of primitives. So is a list of keyed structs, since a struct is a
value and never an element, but an object list inside a keyed struct is walked like any
other. The walk follows declared member types: a member declared `object` is untyped on
disk, and a subclass's extra members under a non-union base-typed member are not written,
so neither is walked. Whole values are `ImmutableArray<T>`, `ArraySegment<T>`,
`Memory<T>`, `KeyValuePair`, tuples, `Lazy<T>`, a class or struct without
`[MessagePackObject]`, a `[DataContract]` type, a string-keyed type, a type or member
declaring `[MessagePackFormatter]`, and a non-public member of a type without
`AllowPrivate`. The members read are an integer `[Key]` with no `[IgnoreMember]`: a
public field or a property whose getter is public, or any visibility up the base chain
under the most derived `[MessagePackObject(AllowPrivate = true)]` declaration. An id member
needs a setter MessagePack calls (public or `init`; any under `AllowPrivate`), so the id
MessagePack writes is read back. Still refused: a bad id member (missing, duplicate, not
a string, no integer `[Key]`, `[MessagePackFormatter]` on it, one MessagePack does not
write or cannot assign, a key shared with another member, a bad derived source), a
concrete union arm that is not keyed, a keyed type that is also a collection
(`ICollection<T>`, `IList`, `IDictionary`: MessagePack writes it as a collection, so C#
refuses it at the first encode or decode when it is on the path to an object list), a
type MessagePack has no formatter for, and a formatter resolver that claims a type on the
path. A formatter MessagePack's source generator emits is not refused.
## Canonical Semantic Schema Hash

19
src/GameCult.Caching/Contracts/cultcache-store-composition.md
View file

@ -294,6 +294,25 @@ exclusive open of the same lock file exclude each other is not established.
for `X` that `Doc<X>` needs is declared in `Doc`'s assembly.
- An assembly that declares nothing, and the store envelope, use the base
options.
- A resolver may not claim a type a document's element lists pass through: the list's
element types, the types and members on the way to them, and the collection types
the walk passes (`List<T>`, `T[]`, a collection class, a dictionary on the way).
Those are written by their `[Key]` layout so each element's id reaches disk, and the
codec refuses the document on first encode or decode, naming the resolver and the
type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a
type does not claim it: the extra non-public members the `AllowPrivate` formatter writes
are whole values the walk does not own. A type or member declaring
`[MessagePackFormatter]` is a whole value off the path, the supported way to give a type its
own formatter. A value list (`List<float2>`, a list of a non-keyed class) is not on the
path, so a resolver may claim it.
- The codec also checks, for every type on that path, that `StandardResolver` does not write it
against the walk. A type read by key may not be written by the generic collection resolver
(a `[MessagePackObject]` class that is also an `ICollection<T>` or dictionary); it is
refused on first encode or decode, naming the type. Any other formatter, such as one
MessagePack's source generator emits, passes: the walk decides which types carry ids and
the door only verifies. It also refuses, typed, a type MessagePack cannot build a formatter
for (it throws, or `StandardResolver` returns none), and a resolver that throws while asked
about one.
- Deserialization runs under untrusted-data security. `CultRecordRef<T>` is a
permitted dictionary key, compared by its key string.

73
src/GameCult.Caching/CultCache.cs
View file

@ -545,11 +545,14 @@ namespace GameCult.Caching
get
{
if (Variant == null)
return CultElementIds.Holds(_document);
return CultElementIds.Holds(Descriptor.DocumentType, _document);
if (_codec == null)
return true;
return Variant.Overrides.Any(entry => CultElementIds.Holds(_codec.Deserialize(
Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType, Descriptor.DocumentType, entry.Value)));
return Variant.Overrides.Any(entry =>
{
var memberType = Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType;
return CultElementIds.Holds(memberType, _codec.Deserialize(memberType, Descriptor.DocumentType, entry.Value));
});
}
}
@ -2734,7 +2737,7 @@ namespace GameCult.Caching
// again, and lands in memory only (variants mint inside Resolve, into their override values).
var idPlans = admitted
.Where(stored => stored.Variant == null)
.Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Document, stored.Key.Value, deterministic: source != null)))
.Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Descriptor.DocumentType, stored.Document, stored.Key.Value, deterministic: source != null)))
.ToArray();
VariantPlan plan;
CultCommitOutcome outcome;
@ -3176,7 +3179,7 @@ namespace GameCult.Caching
// The override's value is minted like a plain record's members: ids the store never held are filled here, in the
// decoded copy, and the delta this hold lands carries them.
var bytes = entry.Value;
var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null);
var idPlan = CultElementIds.Plan(member.MemberType, value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null);
if (idPlan.Count > 0)
{
idPlan.Apply();
@ -3491,6 +3494,9 @@ namespace GameCult.Caching
protected CultDocumentRegistry Registry =>
_registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry.");
// Null for a store no registry was attached to (CultMesh's raw single-file write).
private protected CultDocumentRegistry? AttachedRegistry => _registry;
protected ConcurrentDictionary<string, CultStoredDocument> Entries { get; } =
new(StringComparer.Ordinal);
@ -3542,13 +3548,16 @@ namespace GameCult.Caching
return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1;
}
// The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an
// element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any
// The header a write leaves, for either store kind: HeaderFor decides. An element id is an ordinary keyed string, so only a
// reader with the record's type can tell from its bytes whether it holds one; the store header is how a reader without the types
// learns it. A write that copies any
// record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An
// opaque payload (null) never marks an unmarked store, and keeps a marked one marked.
protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection<bool?> written, bool copiesAny, bool directoryStore, bool holdsVariants) =>
// opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes
// under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may
// have written ids under an unmarked header. Nothing a load or a decode records can unmark a store.
protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection<bool?> written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) =>
HeaderFor(
holdsIds: written.Any(value => value == true),
holdsIds: copiedHoldIds || written.Any(value => value == true),
durableHeader,
wholeStore: !copiesAny && written.All(value => value != null),
directoryStore,
@ -3968,6 +3977,9 @@ namespace GameCult.Caching
// Test seam: how many record encodings this store has hashed, to prove what a write hashes.
internal int RecordsHashed;
// Test seam: how many copied records a write decoded to ask whether they hold an element id.
internal int CopiedRecordsDecoded;
private string VersionOfBytes(byte[] bytes)
{
RecordsHashed++;
@ -4206,7 +4218,8 @@ namespace GameCult.Caching
// write that then has nothing to write but keys it checked writes no file. Then a store that holds
// a variant, or would after this write, is written only while its record versions are what this store last read: a variant
// resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions
// only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is
// only: a write hashes other records' bytes, never decodes them. That holds for the variant gate; the one decode of a copied
// record is the element-id check (CopiedHoldIds), which runs only on an unmarked store and can only mark it. Otherwise the whole write is refused and what it refused is
// reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that
// encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of
// the bytes written.
@ -4250,6 +4263,7 @@ namespace GameCult.Caching
durable?.Snapshot.FormatVersion,
upserts.Select(staged => staged.HoldsIds).ToArray(),
copiesAny: copied.Length > 0,
copiedHoldIds: CopiedHoldIds(durable, copied, upserts),
directoryStore: false,
holdsVariants: records.Any(record => record.Variant != null));
@ -4263,6 +4277,43 @@ namespace GameCult.Caching
return versions;
}
// Whether a record this write copies forward holds an element id, decided from the bytes the file holds under the store's
// lock, so every C# writer with a type for the record decides the same from the same bytes. Positive-only: it runs only
// while the header on disk is unmarked and can only mark the store, so a narrower type that decodes no ids changes nothing.
// It never refuses a write: a record that does not resolve or decode is copied as its bytes and decides nothing. Gates, in
// order, so the decode runs only where it can change the answer.
private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection<CultStagedRecord> upserts)
{
if (durable == null ||
string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) ||
upserts.Any(staged => staged.HoldsIds == true) ||
AttachedRegistry is not { } registry)
return false;
var catalog = durable.Snapshot.SchemaCatalog;
foreach (var record in copied)
{
if (record.Variant != null)
continue;
try
{
if (registry.Claimant(record.SchemaId, catalog, out _) == null)
continue;
var type = registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog).Descriptor.DocumentType;
if (!CultElementIds.CanHoldIds(type))
continue;
CopiedRecordsDecoded++;
if (CultElementIds.Holds(type, DeserializePayload(type, record.Payload)))
return true;
}
catch (Exception e) when (e is not OperationCanceledException)
{
// Copied as its bytes: no type, or no decode, is no fact.
}
}
return false;
}
private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection<CultStagedRecord> upserts, IReadOnlyCollection<string> removals)
{
var records = durable?.Snapshot.Records ?? Array.Empty<CultPersistedRecord>();

394
src/GameCult.Caching/CultElementIds.cs
View file

@ -5,6 +5,7 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using System.Text;
using MessagePack;
@ -85,7 +86,7 @@ namespace GameCult.Caching
public Action<object, string>? SetId;
public Func<object, string?>? Derive;
public string? DerivedFrom;
public (int Slot, Func<object, object?> Get)[] Walk = Array.Empty<(int, Func<object, object?>)>();
public (int Slot, Type Type, Func<object, object?> Get)[] Walk = Array.Empty<(int, Type, Func<object, object?>)>();
}
private static readonly ConcurrentDictionary<Type, Shape> Shapes = new();
@ -95,57 +96,144 @@ namespace GameCult.Caching
internal sealed class Survey
{
public List<string> Problems { get; } = new();
// Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's
// element types and every concrete type under them. A serialization codec must write all of these by their [Key]
// layout, so it checks them against the formatters it composed.
public List<Type> IdPath { get; } = new();
// The concrete element types already judged in this document, so each is refused at most once however many lists hold it.
internal HashSet<Type> Judged { get; } = new();
}
// What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type
// can sit in a cycle, so the path is a fixed point over these edges and not a stack.
private sealed class Graph
{
public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new();
public List<(Type Owner, MemberInfo Member, Type Element, List<Type> Concrete, Type[] Through)> Lists { get; } = new();
// Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too.
public List<Type> Enumerating { get; } = new();
}
private static readonly ConcurrentDictionary<Type, bool> IdCapable = new();
// A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask.
internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0);
// Every reason this document type cannot be registered under the id rule, one per offending concrete element type.
internal static Survey Inspect(Type documentType)
{
var survey = new Survey();
VisitObject(documentType, documentType, new HashSet<Type>(), survey);
var graph = new Graph();
VisitObject(documentType, documentType, new HashSet<Type>(), survey, graph);
Settle(documentType, survey, graph);
return survey;
}
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, Survey survey)
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, Survey survey, Graph graph)
{
if (!seen.Add(type))
return;
if (EnumeratesObjects(type))
graph.Enumerating.Add(type);
foreach (var (member, memberType) in KeyedMembers(type))
VisitMemberType(documentType, type, member, memberType, seen, survey);
VisitMemberType(documentType, type, member, memberType, Array.Empty<Type>(), seen, survey, graph);
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
VisitObject(documentType, union.SubType, seen, survey);
{
graph.Edges.Add((type, null, union.SubType, Array.Empty<Type>()));
VisitObject(documentType, union.SubType, seen, survey, graph);
}
}
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet<Type> seen, Survey survey)
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet<Type> seen, Survey survey, Graph graph)
{
memberType = Nullable.GetUnderlyingType(memberType) ?? memberType;
if (IsLeaf(memberType))
// A sequence or map whose element is itself (an unkeyed Node : IEnumerable<Node>) ends the type walk here; without
// this it would recurse until the stack overflows.
if (Array.IndexOf(through, memberType) >= 0)
return;
if (DictionaryValueType(memberType) is { } valueType)
var (kind, inner) = Classify(memberType);
switch (kind)
{
VisitMemberType(documentType, owner, member, valueType, seen, survey);
return;
}
if (ElementTypeOf(memberType) is { } element)
{
if (IsObjectType(element))
{
foreach (var concrete in ConcreteTypes(element))
case Kind.Leaf:
// A shape the walk does not own is one whole value, never refused (q3 amended).
return;
case Kind.Map:
VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph);
return;
case Kind.Sequence when IsElement(inner):
var concretes = ConcreteTypes(inner).ToList();
graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray()));
foreach (var concrete in concretes)
{
var problem = IdProblem(concrete);
if (!survey.Judged.Add(concrete))
continue;
var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem;
if (problem != null)
survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}");
}
VisitObject(documentType, element, seen, survey);
VisitObject(documentType, inner, seen, survey, graph);
return;
case Kind.Sequence:
VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph);
return;
default:
graph.Edges.Add((owner, member, memberType, through));
VisitObject(documentType, memberType, seen, survey, graph);
return;
}
}
private const string UnkeyedElementProblem =
"is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists.";
// The id path: every type that reaches an object list through members MessagePack writes by key, the lists' element and
// concrete types, and the keyed types that enumerate objects. The codec verifies that MessagePack writes each of them the
// way Classify reads it; it never decides ownership.
private static void Settle(Type documentType, Survey survey, Graph graph)
{
var lists = graph.Lists;
var reaching = new HashSet<Type>(lists.Select(list => list.Owner));
var through = new List<Type>(lists.SelectMany(list => list.Through));
for (var grew = true; grew;)
{
grew = false;
foreach (var (owner, _, child, via) in graph.Edges)
{
if (!reaching.Contains(child))
continue;
through.AddRange(via);
grew |= reaching.Add(owner);
}
else
VisitMemberType(documentType, owner, member, element, seen, survey);
return;
}
if (IsObjectType(memberType))
VisitObject(documentType, memberType, seen, survey);
var path = new List<Type>();
void Add(Type type)
{
if (!path.Contains(type))
path.Add(type);
}
foreach (var type in reaching)
Add(type);
foreach (var type in through)
Add(type);
foreach (var list in lists)
{
Add(list.Element);
foreach (var concrete in list.Concrete)
Add(concrete);
}
// A keyed type the walk reads by key but whose enumeration holds objects is on the path: the codec asks whether
// MessagePack writes it by key too.
foreach (var type in graph.Enumerating)
Add(type);
survey.IdPath.AddRange(path);
}
// The types an element can actually be: the element type itself when it can be instantiated, and every union subtype
@ -171,21 +259,31 @@ namespace GameCult.Caching
private static string? IdProblem(Type element)
{
var ids = MembersOf(element).Where(member => member.IsDefined(typeof(CultElementIdAttribute), true)).ToArray();
// Candidates come from every instance member up the base chain, public or not, so an id MessagePack does not write
// is named as such instead of being reported missing. An override or a hiding member is one member, the most derived.
var ids = AllInstanceMembers(element).Where(member => Declares(member, typeof(CultElementIdAttribute)))
.GroupBy(member => member.Name).Select(group => group.First()).ToArray();
if (ids.Length == 0)
return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity.";
if (ids.Length > 1)
return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity.";
var id = ids[0];
if (Declares(id, typeof(MessagePackFormatterAttribute)))
return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key].";
if (TypeOf(id) != typeof(string))
return $"declares [CultElementId] on {id.Name}, which is not a string.";
if (KeyOf(id) is not { } idKey)
return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted.";
if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written)
return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], or non-public on a type without [MessagePackObject(AllowPrivate = true)]); the id would never reach disk.";
id = written;
var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey);
if (sharing.Member != null)
return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own.";
if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true })
return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it.";
if (id is PropertyInfo { SetMethod.IsPublic: false } && !AllowsPrivate(element))
return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine).";
var source = id.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom;
if (source != null)
{
@ -203,40 +301,37 @@ namespace GameCult.Caching
// ---- write: what the stored form holds ----
// True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value
// as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write.
internal static bool Holds(object? root)
// as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. The walk
// follows declared types, as registration does; a value's own type picks only a union arm.
internal static bool Holds(Type declared, object? root)
{
if (root == null)
return false;
var type = root.GetType();
if (IsLeaf(type))
return false;
if (root is IDictionary dictionary)
var (kind, inner) = Classify(declared);
switch (kind)
{
foreach (DictionaryEntry entry in dictionary)
{
if (Holds(entry.Value))
return true;
}
case Kind.Leaf:
return false;
case Kind.Map:
return MapEntries(declared)(root).Any(entry => Holds(inner, entry.Value));
case Kind.Sequence:
var elements = IsElement(inner);
if (!elements && Classify(inner).Kind == Kind.Leaf)
return false;
foreach (var item in (IEnumerable)root)
{
if (item == null)
continue;
if (elements && OwnerShape(inner, item).GetId is { } getId && !string.IsNullOrEmpty(getId(item)))
return true;
if (Holds(inner, item))
return true;
}
return false;
return false;
default:
return OwnerShape(declared, root).Walk.Any(member => Holds(member.Type, member.Get(root)));
}
if (root is IEnumerable list)
{
foreach (var item in list)
{
if (item != null && IsObjectType(item.GetType()) && ShapeOf(item.GetType()).GetId is { } getId &&
!string.IsNullOrEmpty(getId(item)))
return true;
if (Holds(item))
return true;
}
return false;
}
return IsObjectType(type) && ShapeOf(type).Walk.Any(member => Holds(member.Get(root)));
}
// ---- write and load: mint and check ----
@ -245,41 +340,48 @@ namespace GameCult.Caching
// that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty.
// rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits
// twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load.
internal static IdPlan Plan(object? root, string rootPath, bool deterministic)
internal static IdPlan Plan(Type declared, object? root, string rootPath, bool deterministic)
{
var plan = new IdPlan();
Visit(root, rootPath);
Visit(declared, root, rootPath);
return plan;
void Visit(object? value, string path)
void Visit(Type type, object? value, string path)
{
if (value == null)
return;
var type = value.GetType();
if (IsLeaf(type))
return;
if (value is IDictionary dictionary)
var (kind, inner) = Classify(type);
switch (kind)
{
foreach (DictionaryEntry entry in dictionary)
Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}");
}
else if (value is IEnumerable list)
Elements(list, path);
else if (IsObjectType(type))
{
foreach (var (slot, get) in ShapeOf(type).Walk)
Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture));
case Kind.Map:
foreach (var entry in MapEntries(type)(value))
Visit(inner, entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}");
break;
case Kind.Sequence when IsElement(inner):
Elements(inner, (IEnumerable)value, path);
break;
case Kind.Sequence:
if (Classify(inner).Kind == Kind.Leaf)
break;
var index = 0;
foreach (var item in (IEnumerable)value)
Visit(inner, item, path + "/" + (index++).ToString(CultureInfo.InvariantCulture));
break;
case Kind.Owner:
foreach (var (slot, memberType, get) in OwnerShape(type, value).Walk)
Visit(memberType, get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture));
break;
}
}
void Elements(IEnumerable list, string path)
void Elements(Type element, IEnumerable list, string path)
{
var items = list.Cast<object?>().ToArray();
var taken = new HashSet<string>(StringComparer.Ordinal);
var objects = new HashSet<object>(ReferenceComparer.Instance);
foreach (var item in items)
{
if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape)
if (item == null || OwnerShape(element, item) is not { GetId: { } getId } shape)
continue;
if (!objects.Add(item))
throw new CultElementIdException(
@ -302,7 +404,7 @@ namespace GameCult.Caching
if (item == null)
continue;
var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture);
if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape &&
if (OwnerShape(element, item) is { SetId: { } , GetId: { } get } itemShape &&
string.IsNullOrEmpty(get(item)))
{
string id;
@ -321,7 +423,7 @@ namespace GameCult.Caching
plan.Add(item, itemShape, get(item), id);
}
Visit(item, itemPath);
Visit(element, item, itemPath);
}
}
@ -389,10 +491,18 @@ namespace GameCult.Caching
private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build);
// The shape of an owner or element a walk holds under a declared type: a union base writes its arm, so the value's own
// type picks the arm; any other declared type is written by its own [Key] layout, whatever a subclass adds.
private static Shape OwnerShape(Type declared, object value)
{
declared = Nullable.GetUnderlyingType(declared) ?? declared;
return declared.IsDefined(typeof(UnionAttribute), false) ? ShapeOf(value.GetType()) : ShapeOf(declared);
}
private static Shape Build(Type type)
{
var shape = new Shape();
var idMember = MembersOf(type).FirstOrDefault(member => member.IsDefined(typeof(CultElementIdAttribute), true));
var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute)));
if (idMember != null)
{
shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o);
@ -406,20 +516,62 @@ namespace GameCult.Caching
}
shape.Walk = KeyedMembers(type)
.Where(entry => !IsLeaf(Nullable.GetUnderlyingType(entry.Type) ?? entry.Type))
.Select(entry => (Slot: KeyOf(entry.Member)!.Value, Get: entry.Member is FieldInfo fi
.Where(entry => Classify(entry.Type).Kind != Kind.Leaf)
.Select(entry => (Slot: KeyOf(entry.Member)!.Value, entry.Type, Get: entry.Member is FieldInfo fi
? (Func<object, object?>)(o => fi.GetValue(o))
: o => ((PropertyInfo)entry.Member).GetValue(o)))
.ToArray();
return shape;
}
// The members MessagePack's StandardResolver writes for an integer-keyed type, none of them [IgnoreMember]. Without
// AllowPrivate: public instance fields, and public instance properties with a public getter. With it, read from the most
// derived [MessagePackObject] declaration (a derived type without it is written without it, probe NoApDerived): every
// instance field and indexer-free property up the base chain at any visibility, the most derived of a name.
// TheWalkReadsTheMembersMessagePackWrites checks this set against MessagePack's own member discovery.
private static IEnumerable<MemberInfo> MembersOf(Type type) =>
AllowsPrivate(type)
? AllInstanceMembers(type).Where(member => !Declares(member, typeof(IgnoreMemberAttribute)))
.GroupBy(member => member.Name).Select(group => group.First())
: PublicMembersOf(type);
private static bool AllowsPrivate(Type type) => type.GetCustomAttribute<MessagePackObjectAttribute>(true)?.AllowPrivate == true;
private static IEnumerable<MemberInfo> PublicMembersOf(Type type) =>
type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast<MemberInfo>()
.Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod != null));
.Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public)
.Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true }))
.Where(member => !Declares(member, typeof(IgnoreMemberAttribute)));
// Every instance field and indexer-free property declared on the type or any base, most derived first.
private static IEnumerable<MemberInfo> AllInstanceMembers(Type type)
{
const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly;
for (var level = type; level != null && level != typeof(object); level = level.BaseType)
{
foreach (var field in level.GetFields(flags))
yield return field;
foreach (var property in level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0))
yield return property;
}
}
// The [Key] members MessagePack writes, for the oracle test that checks them against MessagePack.
internal static IEnumerable<MemberInfo> WrittenKeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null);
private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) =>
MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member)));
WrittenKeyedMembers(type).Where(member => !Declares(member, typeof(MessagePackFormatterAttribute)))
.Select(member => (member, TypeOf(member)));
// An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an
// override of a base virtual member would not see its base's attribute.
private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true);
// True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it.
internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner;
private static bool EnumeratesObjects(Type type) =>
(MapTypes(type)?.Value ?? ElementTypeOf(type)) is { } inner && Classify(inner).Kind != Kind.Leaf;
private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute<KeyAttribute>(true)?.IntKey;
@ -427,18 +579,57 @@ namespace GameCult.Caching
// ---- type classification ----
// A value MessagePack writes whole and that cannot hold an element list. Structs are values: never elements, never walked.
private static bool IsLeaf(Type type)
{
if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type.IsValueType)
return true;
return type.IsArray && IsLeaf(type.GetElementType()!);
}
private enum Kind { Leaf, Map, Sequence, Owner }
// An object a store can hold an element list under, or hold as an element: a class MessagePack serializes by key.
private static bool IsObjectType(Type type) =>
(type.IsClass || type.IsInterface) && type != typeof(string) && type != typeof(object) &&
(type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true));
private static readonly ConcurrentDictionary<Type, (Kind Kind, Type Inner)> Classes = new();
// The walk's model of a declared type: registration, Plan, Holds and the codec's id path all go through it. A
// [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable<T>; the codec checks, for every type
// on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not.
// Inner is a map's value type, a sequence's element type, or the type itself for an owner.
// The owned shapes are a closed list (Owner, Sequence, Map); every other shape is Leaf: one whole value with no element
// ids, replaced whole by a variant (q3 amended 2026-10-16). When in doubt the answer is Leaf.
// Leaf strings (an IEnumerable<char> MessagePack writes whole), a type written whole (IsWrittenWhole: attribute-
// formatted, [DataContract], string-keyed), a struct or a class or interface with no [MessagePackObject] or
// [Union] and no collection shape (primitives, enums, float2, DateTime, ImmutableArray, ArraySegment, Memory,
// KeyValuePair, tuples, Lazy, unkeyed classes), and object, which is untyped on disk.
// Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>.
// Sequence an array or an IEnumerable<T> class or interface.
// Owner a class, interface or struct MessagePack serializes by integer key. A keyed struct is walked as an owner;
// it is never an element (IsElement), since a boxed copy cannot carry a minted id.
private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type =>
{
type = Nullable.GetUnderlyingType(type) ?? type;
if (type == typeof(string))
return (Kind.Leaf, type);
if (IsWrittenWhole(type))
return (Kind.Leaf, type);
if (IsKeyed(type))
return (Kind.Owner, type);
if (type.IsValueType)
return (Kind.Leaf, type);
if (MapTypes(type) is { } map)
return (Kind.Map, map.Value);
if (ElementTypeOf(type) is { } element)
return (Kind.Sequence, element);
return (Kind.Leaf, type);
});
// A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type,
// inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string
// keys (KeyAsPropertyName, or a [Key] with a string key on a member MessagePack writes: a private or [IgnoreMember] one is not written). Such a type is Leaf, even when it is also enumerable.
private static bool IsWrittenWhole(Type type) =>
type.IsDefined(typeof(MessagePackFormatterAttribute), true) ||
(type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) ||
type.GetCustomAttribute<MessagePackObjectAttribute>(true)?.KeyAsPropertyName == true ||
MembersOf(type).Any(member => member.GetCustomAttribute<KeyAttribute>(true)?.StringKey != null);
// A type MessagePack serializes by key: [MessagePackObject] or [Union].
private static bool IsKeyed(Type type) =>
type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true);
// What an object list holds: a keyed class or interface. Never a struct.
private static bool IsElement(Type type) => !type.IsValueType && Classify(type).Kind == Kind.Owner;
private static Type? ElementTypeOf(Type type)
{
@ -453,15 +644,34 @@ namespace GameCult.Caching
return null;
}
private static Type? DictionaryValueType(Type type)
// The key and value types of a map MessagePack writes as a map: IDictionary<,> or IReadOnlyDictionary<,>.
private static (Type Key, Type Value)? MapTypes(Type type)
{
foreach (var candidate in type.GetInterfaces().Append(type))
{
if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>))
return candidate.GetGenericArguments()[1];
if (candidate.IsGenericType &&
(candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>) || candidate.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>)))
{
var arguments = candidate.GetGenericArguments();
return (arguments[0], arguments[1]);
}
}
return null;
}
private static readonly ConcurrentDictionary<Type, Func<object, IEnumerable<(object? Key, object? Value)>>> MapReaders = new();
// A map's entries read through the generic interface Classify found, so no map shape is recognised twice.
private static Func<object, IEnumerable<(object? Key, object? Value)>> MapEntries(Type map) => MapReaders.GetOrAdd(map, static type =>
{
var (key, value) = MapTypes(type)!.Value;
return (Func<object, IEnumerable<(object? Key, object? Value)>>)Delegate.CreateDelegate(
typeof(Func<object, IEnumerable<(object? Key, object? Value)>>),
typeof(CultElementIds).GetMethod(nameof(ReadMap), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(key, value));
});
private static IEnumerable<(object? Key, object? Value)> ReadMap<TKey, TValue>(object map) =>
((IEnumerable<KeyValuePair<TKey, TValue>>)map).Select(entry => ((object?)entry.Key, (object?)entry.Value));
}
}

9
src/GameCult.Caching/README.md
View file

@ -195,6 +195,15 @@ A document type that holds a list of objects gives every object type in that lis
`[Key]`. The id is what lets a variant override one list element instead of the
whole list. Lists of strings, numbers and other non-object values need none.
The rule covers the shapes the id walk owns: arrays, `IEnumerable<T>` classes and
interfaces, dictionary values, and integer-keyed `[MessagePackObject]` and `[Union]`
types. Any other holder is one whole value: `ImmutableArray<T>`, `ArraySegment<T>`,
`Memory<T>`, `KeyValuePair`, tuples, `Lazy<T>`, a class without `[MessagePackObject]`,
a `[DataContract]` or string-keyed type, and a type or member declaring
`[MessagePackFormatter]`. Its elements carry no element ids: a `[CultElementId]`
member inside it is a plain string, never minted or checked, and a variant can only
replace the holder whole.
```csharp
[MessagePackObject]
public sealed class Line

21
src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
View file

@ -98,7 +98,7 @@ namespace GameCult.Mesh
var descriptor = resolvedRegistry.GetRequired<TDocument>();
// Element ids are decided as a cache write decides them, before anything is written: an unset id is minted, a
// refused list refuses the write, and the file is marked when it holds an id. A failed write gives the ids back.
var ids = CultElementIds.Plan(document, key.Value, deterministic: false);
var ids = CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: false);
ids.Apply();
try
{
@ -108,7 +108,8 @@ namespace GameCult.Mesh
descriptor.ToCatalogEntry(),
storedAt,
CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)),
CultElementIds.Holds(document));
CultElementIds.Holds(typeof(TDocument), document),
resolvedRegistry);
}
catch
{
@ -131,7 +132,7 @@ namespace GameCult.Mesh
var payload = ReadSingleFileDocumentPayload(path, key, descriptor.SchemaId);
var document = (TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), payload);
// A load refuses what a write refuses, and mints the ids a cache load of the same file would mint.
CultElementIds.Plan(document, key.Value, deterministic: true).Apply();
CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: true).Apply();
return document;
}
@ -146,7 +147,7 @@ namespace GameCult.Mesh
byte[] payload)
{
if (schema == null) throw new ArgumentNullException(nameof(schema));
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null);
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null);
}
/// <summary>
@ -229,7 +230,8 @@ namespace GameCult.Mesh
CultSchemaCatalogEntry catalogEntry,
string? storedAt,
byte[] payload,
bool? holdsIds)
bool? holdsIds,
CultDocumentRegistry? registry)
{
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry));
@ -239,8 +241,13 @@ namespace GameCult.Mesh
payload ??= Array.Empty<byte>();
// The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under
// the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null),
// so a file already marked stays marked. A store this runtime cannot read refuses the write instead of being overwritten.
StoreAt(path).ApplyWriteSet(
// so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write
// decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime
// cannot read refuses the write instead of being overwritten.
var store = StoreAt(path);
if (registry != null)
store.AttachRegistry(registry);
store.ApplyWriteSet(
new[]
{
new CultStagedRecord(

3
tests/GameCult.Caching.Tests/AssemblyInfo.cs
View file

@ -6,7 +6,10 @@ using MessagePack.Formatters;
[assembly: CultCacheFormatterResolver(typeof(PairResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))]
namespace GameCult.Caching.Tests
{

730
tests/GameCult.Caching.Tests/ElementIdEngineTests.cs
View file

@ -1,7 +1,9 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
using GameCult.Caching;
using MessagePack;
@ -16,7 +18,7 @@ namespace GameCult.Caching.Tests
private static int Assign(object root, string key, bool deterministic)
{
var plan = CultElementIds.Plan(root, key, deterministic);
var plan = CultElementIds.Plan(root.GetType(), root, key, deterministic);
plan.Apply();
return plan.Count;
}
@ -30,7 +32,8 @@ namespace GameCult.Caching.Tests
foreach (var holder in new[]
{
typeof(HList), typeof(HArray), typeof(HDictionary), typeof(HNested), typeof(HListOfLists),
typeof(HElementNest), typeof(HUnion), typeof(HNullableInner)
typeof(HElementNest), typeof(HUnion), typeof(HNullableInner), typeof(HStruct), typeof(HStructList), typeof(HNullableStruct),
typeof(HReadOnlyMap)
})
RefusesBare(holder);
Assert.That(Problems(typeof(HFine)), Is.Empty);
@ -72,28 +75,28 @@ namespace GameCult.Caching.Tests
public void HoldsFindsASetIdWhereverAListCanSitAndNothingElse()
{
const string id = "0123456789ab";
Assert.That(CultElementIds.Holds(null), Is.False);
Assert.That(CultElementIds.Holds(new HAll
Assert.That(CultElementIds.Holds(typeof(HAll), null), Is.False);
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll
{
Dict = { ["k"] = new HHasIds { Items = { new EIded() } } },
Grid = { new List<EIded> { new EIded() } },
Nest = { new ENest { Sub = { new EIded() } } },
}), Is.False, "unset ids hold nothing");
Assert.That(CultElementIds.Holds(new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value");
Assert.That(CultElementIds.Holds(new HAll { Grid = { new List<EIded> { new EIded { Id = id } } } }), Is.True, "a list of lists");
Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element");
Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element");
Assert.That(CultElementIds.Holds(new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value");
Assert.That(CultElementIds.Holds(new HAll { Grid = { new List<EIded> { new EIded() }, new List<EIded> { new EIded { Id = id } } } }), Is.True, "the second list");
Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements");
Assert.That(CultElementIds.Holds(new HAll { Grid = { new List<EIded> { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List<EIded> { new EIded { Id = id } } } }), Is.True, "a list of lists");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List<EIded> { new EIded() }, new List<EIded> { new EIded { Id = id } } } }), Is.True, "the second list");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements");
Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List<EIded> { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list");
}
[Test]
public void APlanCountsAndChangesNothingUntilApplied()
{
var root = new HHasIds { Items = { new EIded(), new EIded() } };
var plan = CultElementIds.Plan(root, "r", deterministic: true);
var plan = CultElementIds.Plan(typeof(HHasIds), root, "r", deterministic: true);
Assert.That(plan.Count, Is.EqualTo(2));
Assert.That(root.Items.Select(x => x.Id), Has.All.Empty);
plan.Apply();
@ -184,7 +187,7 @@ namespace GameCult.Caching.Tests
Assert.That(root.Dict["k"].Items.Select(x => x.Id), Has.All.Empty, "the list before the refused one was not minted into");
var ok = new HHasIds { Items = { new EIded(), new EIded { Id = "dddddddddddd" } } };
var plan = CultElementIds.Plan(ok, "r", deterministic: false);
var plan = CultElementIds.Plan(typeof(HHasIds), ok, "r", deterministic: false);
plan.Apply();
plan.Undo();
Assert.That(ok.Items.Select(x => x.Id), Is.EqualTo(new[] { "", "dddddddddddd" }));
@ -290,12 +293,637 @@ namespace GameCult.Caching.Tests
Assert.That(Assert.Throws<CultElementIdException>(() => Assign(twice, "r", deterministic: false))!.Message, Does.Contain("'5'"));
}
// A list element is written by its [Key] layout or its id never reaches disk. A union arm without [MessagePackObject]
// is written by some other formatter; a list of a type that is not keyed at all is a value list, replaced whole.
[Test]
public void ANonKeyedArmOfAListUnionIsRefusedAtRegistration()
{
var problem = Problems(typeof(HUnkeyedArmList)).Single();
Assert.That(problem, Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own").And.Contain("HUnkeyedArmList"));
Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty, "the same union as a field has no element id to lose");
Assert.That(Problems(typeof(HValueList)), Is.Empty, "a list of a non-keyed type is a value list");
}
// [MessagePackFormatter] on an element type, an owner, a collection class or a member makes that value one whole value: it
// has no element ids and is off the id path. The attribute on the id member itself is the one refusal that stays.
[Test]
public void AFormatterAttributeMakesAWholeValueExceptOnTheId()
{
Assert.That(Problems(typeof(HFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]"));
foreach (var (type, holder) in new (Type, object)[]
{
(typeof(HFormattedElement), new HFormattedElement { Items = { new FormattedElement() } }),
(typeof(HFormattedList), new HFormattedList { Items = { new EIded() } }),
(typeof(HFormattedOwner), new HFormattedOwner { Owner = { Items = { new EIded() } } }),
(typeof(HFormattedMember), new HFormattedMember { Inner = { Items = { new EIded() } } }),
(typeof(HFormattedDerived), new HFormattedDerived { Items = { new FormattedDerived() } }),
})
{
Assert.That(Problems(type), Is.Empty, type.Name);
Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name);
Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(0), type.Name);
}
Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field");
}
// The collection types between a member and its object list are written by some formatter too, so they sit on the id path.
// A value list (of a struct, a primitive or a non-keyed class) never reaches an object, so it does not.
[Test]
public void ACollectionTypeOnTheWayToAnObjectListIsOnTheIdPath()
{
var path = CultElementIds.Inspect(typeof(HCollections)).IdPath;
Assert.That(path, Is.EquivalentTo(new[]
{
typeof(HCollections), typeof(EIded), typeof(HHasIds),
typeof(List<EIded>), typeof(EIded[]), typeof(Dictionary<string, HHasIds>), typeof(List<List<EIded>>)
}));
var valueLists = CultElementIds.Inspect(typeof(HValueListsBesideObjects)).IdPath;
Assert.That(valueLists, Is.EquivalentTo(new[] { typeof(HValueListsBesideObjects), typeof(List<EIded>), typeof(EIded) }));
}
[Test]
public void AFormatterOnACollectionClassOfAnObjectListIsAWholeValue()
{
Assert.That(Problems(typeof(HRoles)), Is.Empty);
Assert.That(Assign(new HRoles { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "its elements carry no ids and are never minted");
Assert.That(Problems(typeof(HFormattedValueCollection)), Is.Empty, "a collection class of a non-keyed element is a value list");
}
// MemberInfo.IsDefined ignores inherit for properties: an override sees its base's attribute only through Declares.
[Test]
public void AnInheritedFormatterOnAnOverriddenMemberIsHonoured()
{
Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]"));
Assert.That(Problems(typeof(OwnerOverFormatted)), Is.Empty, "the override inherits its base's formatter, so the member is a whole value");
Assert.That(CultElementIds.Inspect(typeof(OwnerOverFormatted)).IdPath, Is.Empty, "a walked override would put its list on the id path");
Assert.That(Assign(new OwnerOverFormatted { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "and mint into a list its formatter may drop ids from");
Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id");
var root = new HIdOverride { Items = { new IdOverride() } };
Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(1));
Assert.That(root.Items[0].Id, Has.Length.EqualTo(12));
}
// The union arm has a valid id, so IdProblem passes it and only the rule that an element is written by its [Key] layout
// can refuse it; it is refused once per document however many lists hold it.
[Test]
public void ANonKeyedArmWithAValidIdIsRefusedOncePerType()
{
Assert.That(Problems(typeof(HUnkeyedArmList)).Single(), Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own"));
Assert.That(Problems(typeof(HUnkeyedTwoLists)).Single(), Does.Contain("UnkeyedArm"));
Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty);
}
// The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and
// a type reached twice, are both on the way to it.
[Test]
public void TheIdPathHoldsEveryTypeThatReachesAnObjectList()
{
var path = CultElementIds.Inspect(typeof(HDiamond)).IdPath;
Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded), typeof(List<EIded>) }));
Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded), typeof(List<ENest>), typeof(List<EIded>) }));
Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty);
// A union base is on the path both as the type a list holds (its formatter writes the arm) and as a field type whose
// arm holds the list.
Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm), typeof(List<PlainBase>) }));
Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded), typeof(List<EIded>) }));
}
// A keyed struct is a value MessagePack writes by key, and an object list inside one carries element ids like any other.
// It is walked as an owner and is never an element: a boxed copy cannot carry a minted id.
[Test]
public void AKeyedStructIsWalkedAsAnOwnerAndNeverAnElement()
{
var root = new HStructIded
{
Hold = new SHoldIded { Items = new List<EIded> { new EIded() } },
Many = { new SHoldIded { Items = new List<EIded> { new EIded() } } },
};
Assert.That(Problems(typeof(HStructIded)), Is.Empty);
Assert.That(CultElementIds.Inspect(typeof(HStructIded)).IdPath,
Is.EquivalentTo(new[] { typeof(HStructIded), typeof(SHoldIded), typeof(List<SHoldIded>), typeof(List<EIded>), typeof(EIded) }));
Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.False);
Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(2));
Assert.That(root.Hold.Items[0].Id, Is.Not.Empty, "visible through the struct copy");
Assert.That(root.Many[0].Items[0].Id, Is.Not.Empty, "visible through the struct in a list");
Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.True);
var one = new HStructIded { Hold = new SHoldIded { Items = new List<EIded> { new EIded { Id = "0123456789ab" } } } };
Assert.That(CultElementIds.Holds(typeof(HStructIded), one), Is.True);
Assert.That(Problems(typeof(HStructValues)), Is.Empty);
Assert.That(CultElementIds.Inspect(typeof(HStructValues)).IdPath,
Is.EquivalentTo(new[] { typeof(HStructValues), typeof(List<EIded>), typeof(EIded) }), "a list of keyed structs is a value list");
var values = new HStructValues { Structs = { new SValue { N = 1 } }, Objects = { new EIded() } };
Assert.That(Assign(values, "r", deterministic: false), Is.EqualTo(1));
var nullable = new HNullableStructIded { Hold = new SHoldIded { Items = new List<EIded> { new EIded() } } };
Assert.That(Assign(nullable, "r", deterministic: false), Is.EqualTo(1), "a nullable struct is walked as its struct");
Assert.That(CultElementIds.Holds(typeof(HNullableStructIded), nullable), Is.True);
}
// A type is an owner or an element only when MessagePack serializes it by key ([MessagePackObject], inherited, or
// [Union]). A struct without either is a value however many [Key] members it carries.
[Test]
public void OnlyAKeyedTypeIsWalked()
{
Assert.That(Problems(typeof(HUnkeyedStruct)), Is.Empty);
Assert.That(CultElementIds.Inspect(typeof(HUnkeyedStruct)).IdPath, Is.Empty);
Assert.That(Problems(typeof(HInheritedBare)).Single(), Does.Contain("InheritedBare").And.Contain("[CultElementId]"),
"[MessagePackObject] reaches a subclass through inheritance");
}
// Every dictionary interface MessagePack writes as a map is walked by one path, and the minted path text is the one a
// store loaded before ids already holds, so its deterministic ids do not move.
[Test]
public void EveryMapInterfaceIsWalkedByTheSamePath()
{
var element = new Func<HHasIds>(() => new HHasIds { Items = { new EIded() } });
var viaInterface = new HReadOnlyIded { Map = new Dictionary<string, HHasIds> { ["k"] = element() } };
var viaClass = new HReadOnlyIded { Map = new RoMap(new Dictionary<string, HHasIds> { ["k"] = element() }) };
var viaDictionary = new HMapIded { Map = { ["k"] = element() } };
Assert.That(Assign(viaInterface, "r", deterministic: true), Is.EqualTo(1));
Assert.That(Assign(viaClass, "r", deterministic: true), Is.EqualTo(1), "a class implementing only IReadOnlyDictionary<,>");
Assert.That(Assign(viaDictionary, "r", deterministic: true), Is.EqualTo(1));
var id = viaDictionary.Map["k"].Items[0].Id;
Assert.That(viaInterface.Map["k"].Items[0].Id, Is.EqualTo(id));
Assert.That(viaClass.Map["k"].Items[0].Id, Is.EqualTo(id));
// SHA-256 of "<record key>|<element path>|0": slot 0, the key in braces, slot 0, the index.
var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("r|r.0{k}.0/0|0"));
Assert.That(id, Is.EqualTo(Convert.ToHexString(hash, 0, 6).ToLowerInvariant()), "the path text is unchanged");
var held = new HReadOnlyIded { Map = new RoMap(new Dictionary<string, HHasIds> { ["k"] = new HHasIds { Items = { new EIded { Id = "0123456789ab" } } } }) };
Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), held), Is.True);
Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), viaInterface), Is.True);
Assert.That(CultElementIds.Holds(typeof(HMapIded), new HMapIded { Map = { ["k"] = new HHasIds { Items = { new EIded() } } } }), Is.False);
}
// Registration judges the declared shape, so minting and reading follow it: a member declared object is untyped on disk,
// and what a subclass adds under a base-typed member is not written, so neither is walked.
[Test]
public void TheWalkFollowsTheDeclaredShape()
{
var derived = new HBase { B = new DerC { L = { new EIded(), new EIded { Id = "0123456789ab" } } } };
Assert.That(CultElementIds.Plan(typeof(HBase), derived, "r", deterministic: false).Count, Is.Zero);
Assert.That(CultElementIds.Holds(typeof(HBase), derived), Is.False);
var opaque = new HObject { O = new HHasIds { Items = { new EIded(), new EIded { Id = "0123456789ab" } } } };
Assert.That(CultElementIds.Plan(typeof(HObject), opaque, "r", deterministic: false).Count, Is.Zero);
Assert.That(CultElementIds.Holds(typeof(HObject), opaque), Is.False);
foreach (var type in new[] { typeof(HBase), typeof(HObject) })
{
Assert.That(Problems(type), Is.Empty, type.Name);
Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name);
}
// A union base still writes its arm, so the arm is walked.
var arm = new HIdBaseOwner { Root = new IdArm { Items = { new EIded() } } };
Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1));
}
// MessagePack writes a non-public [Key] member only under AllowPrivate, so a non-public member of a type without it is nobody's
// id business; under AllowPrivate it is written, read and minted like any other member.
[Test]
public void ANonPublicMemberIsWalkedOnlyUnderAllowPrivate()
{
foreach (var type in new[] { typeof(HHiddenList), typeof(HHiddenOwner), typeof(HHiddenFromBase) })
{
Assert.That(Problems(type), Is.Empty, type.Name);
Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name);
}
Assert.That(Problems(typeof(HOptHidden)), Is.Empty);
Assert.That(CultElementIds.Inspect(typeof(HOptHidden)).IdPath, Is.EquivalentTo(new[] { typeof(HOptHidden), typeof(List<EIded>), typeof(EIded) }));
var opt = new HOptHidden();
opt.Peek.Add(new EIded());
opt.Peek.Add(new EIded());
Assert.That(Assign(opt, "r", deterministic: false), Is.EqualTo(2));
Assert.That(opt.Peek.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$"));
Assert.That(Problems(typeof(HHiddenLeaf)), Is.Empty);
Assert.That(CultElementIds.Inspect(typeof(HHiddenLeaf)).IdPath, Is.EquivalentTo(new[] { typeof(HHiddenLeaf), typeof(List<EIded>), typeof(EIded) }),
"a non-public member adds nothing to the id path");
}
// MessagePack writes a [MessagePackObject] class that is only IEnumerable<T> by its keys, so the walk reads it by key. A
// keyed tree node that enumerates its own children is the shape that overflowed the stack when enumeration came first.
[Test]
public void ATypeThatEnumeratesItselfIsWalkedByKeyAndTerminates()
{
var survey = CultElementIds.Inspect(typeof(HKNode));
Assert.That(survey.Problems, Is.Empty);
Assert.That(survey.IdPath, Is.EquivalentTo(new[] { typeof(HKNode), typeof(List<KNode>), typeof(KNode) }));
var root = new HKNode { Nodes = { new KNode { Children = { new KNode { Children = { new KNode() } } } } } };
Assert.That(Assign(root, "r", deterministic: true), Is.EqualTo(3));
Assert.That(root.Nodes[0].Id, Has.Length.EqualTo(12));
Assert.That(root.Nodes[0].Children[0].Children[0].Id, Has.Length.EqualTo(12));
var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;
var back = MessagePackSerializer.Deserialize<HKNode>(MessagePackSerializer.Serialize(root, options), options);
Assert.That(back.Nodes[0].Id, Is.EqualTo(root.Nodes[0].Id), "MessagePack writes the node by key, ids included");
Assert.That(back.Nodes[0].Children[0].Children[0].Id, Is.EqualTo(root.Nodes[0].Children[0].Children[0].Id));
// An unkeyed type that enumerates itself holds no element list and ends the type walk.
var unkeyed = CultElementIds.Inspect(typeof(HUNode));
Assert.That(unkeyed.Problems, Is.Empty);
Assert.That(unkeyed.IdPath, Is.Empty);
// A keyed class that only enumerates is read by its keyed lists, so both are judged and minted.
Assert.That(Problems(typeof(KBag)).Single(), Does.Contain("EBare"));
var bag = new KBagIded { Items = { new EIded(), new EIded() } };
Assert.That(Assign(bag, "r", deterministic: false), Is.EqualTo(2));
Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True);
}
// The walk reads the members MessagePack writes: without AllowPrivate a public field or a property with a public getter, with
// it every member up the base chain, the most derived [MessagePackObject] declaration deciding; no [IgnoreMember].
[Test]
public void TheWalkReadsTheMembersMessagePackWrites()
{
Assert.That(Problems(typeof(PubIgn)), Is.Empty, "an ignored member is never written, so its list is nobody's business");
Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written");
Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate");
foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor), typeof(ApIgnAp), typeof(PSkH), typeof(ISkH) })
{
var allowPrivate = type.GetCustomAttribute<MessagePackObjectAttribute>(true)!.AllowPrivate;
Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name),
Is.EquivalentTo(MessagePackMembers(type, allowPrivate)), type.Name);
}
Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" }),
"the derived declaration decides, so an AllowPrivate base adds nothing");
}
// The names of the members MessagePack's own discovery reads for a type, by reflection into its internal type: the
// oracle the walk's member set is pinned to.
private static List<string> MessagePackMembers(Type type, bool allowPrivate)
{
const string moved = "MessagePack moved its member discovery; re-derive MembersOf";
const System.Reflection.BindingFlags all = System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Instance |
System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic;
var info = typeof(MessagePackSerializer).Assembly.GetType("MessagePack.Internal.ObjectSerializationInfo");
var create = info?.GetMethod("CreateOrNull", all);
if (create == null || create.GetParameters().Length != 4)
{
Assert.Fail(moved);
return new List<string>();
}
var result = create.Invoke(null, new object[] { type, false, false, allowPrivate });
var members = result?.GetType().GetProperty("Members", all)?.GetValue(result) as System.Collections.IEnumerable;
if (members == null)
{
Assert.Fail(moved);
return new List<string>();
}
var names = new List<string>();
foreach (var member in members)
{
var readable = member.GetType().GetProperty("IsReadable", all)?.GetValue(member);
var name = member.GetType().GetProperty("Name", all)?.GetValue(member) as string;
if (readable is not bool || name == null)
Assert.Fail(moved);
if ((bool)readable!)
names.Add(name!);
}
return names;
}
// An id MessagePack writes but cannot read back would be re-minted differently on every load; one it does not write
// never reaches disk.
[Test]
public void AnIdMessagePackCannotReadBackIsRefused()
{
Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id"));
Assert.That(Problems(typeof(HIgnId)).Single(), Does.Contain("does not write").And.Contain("Id"));
Assert.That(Problems(typeof(HPrivateId)).Single(), Does.Contain("does not write").And.Contain("id"));
Assert.That(Problems(typeof(HInit)), Is.Empty);
Assert.That(Problems(typeof(HCtorId)), Is.Empty);
foreach (var holder in new object[]
{
new HInit { Items = { new EInit(), new EInit() } },
new HCtorId { Items = { new ECtorId(null), new ECtorId(null) } },
})
{
Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name);
var bytes = MessagePackSerializer.Serialize(holder.GetType(), holder, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options);
var back = MessagePackSerializer.Deserialize(holder.GetType(), bytes, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options);
Assert.That(Ids(back), Is.EqualTo(Ids(holder)), holder.GetType().Name);
Assert.That(Ids(back), Has.All.Match("^[0-9a-f]{12}$"), holder.GetType().Name);
}
}
// Every shape the walk does not own is one whole value: no problem, off the id path, never minted into and never counted.
[Test]
public void AShapeTheWalkDoesNotOwnIsOneWholeValue()
{
const string Set = "aaaaaaaaaaaa";
EIded E(string id) => new() { Id = id };
var holders = new (Type Type, Func<string, object> Make)[]
{
(typeof(Whole<ImmutableArray<EIded>>), id => new Whole<ImmutableArray<EIded>> { Value = ImmutableArray.Create(E(id)) }),
(typeof(Whole<ImmutableArray<EIded>?>), id => new Whole<ImmutableArray<EIded>?> { Value = ImmutableArray.Create(E(id)) }),
(typeof(Whole<ArraySegment<EIded>>), id => new Whole<ArraySegment<EIded>> { Value = new ArraySegment<EIded>(new[] { E(id) }) }),
(typeof(Whole<Memory<EIded>>), id => new Whole<Memory<EIded>> { Value = new[] { E(id) } }),
(typeof(Whole<ReadOnlyMemory<EIded>>), id => new Whole<ReadOnlyMemory<EIded>> { Value = new[] { E(id) } }),
(typeof(Whole<KeyValuePair<string, List<EIded>>>), id => new Whole<KeyValuePair<string, List<EIded>>> { Value = new("k", new List<EIded> { E(id) }) }),
(typeof(Whole<(List<EIded>, int)>), id => new Whole<(List<EIded>, int)> { Value = (new List<EIded> { E(id) }, 1) }),
(typeof(Whole<Tuple<List<EIded>, int>>), id => new Whole<Tuple<List<EIded>, int>> { Value = Tuple.Create(new List<EIded> { E(id) }, 1) }),
(typeof(Whole<Lazy<List<EIded>>>), id => new Whole<Lazy<List<EIded>>> { Value = new Lazy<List<EIded>>(() => new List<EIded> { E(id) }) }),
(typeof(Whole<NkHold>), id => new Whole<NkHold> { Value = new NkHold { Items = { E(id) } } }),
(typeof(Whole<SkHold>), id => new Whole<SkHold> { Value = new SkHold { Items = { E(id) } } }),
(typeof(Whole<DcHold>), id => new Whole<DcHold> { Value = new DcHold { Items = { E(id) } } }),
(typeof(Whole<KpBase>), id => new Whole<KpBase> { Value = new KpBase { Items = { E(id) } } }),
(typeof(Whole<KpDerived>), id => new Whole<KpDerived> { Value = new KpDerived { Items = { E(id) } } }),
(typeof(HRoles), id => new HRoles { Items = { E(id) } }),
(typeof(HFormattedList), id => new HFormattedList { Items = { E(id) } }),
(typeof(HHiddenList), id => new HHiddenList()),
};
foreach (var (type, make) in holders)
{
Assert.That(Problems(type), Is.Empty, type.ToString());
Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.ToString());
Assert.That(CultElementIds.CanHoldIds(type), Is.False, type.ToString());
Assert.That(Assign(make(string.Empty), "r", deterministic: false), Is.EqualTo(0), type.ToString());
Assert.That(Assign(make(string.Empty), "r", deterministic: true), Is.EqualTo(0), type.ToString());
Assert.That(CultElementIds.Holds(type, make(Set)), Is.False, type.ToString());
}
// A [DataContract] subclass of a [MessagePackObject] type is still read by key: the attribute it inherits keeps it an owner.
Assert.That(CultElementIds.Inspect(typeof(Whole<DcOverMp>)).IdPath, Does.Contain(typeof(List<EIded>)));
// Elements of a string-keyed type carry no id either, and an owned list of a bare element is still refused.
Assert.That(Problems(typeof(Whole<List<SkElem>>)), Is.Empty);
RefusesBare(typeof(Whole<List<EBare>>));
}
// A string key counts only on a member MessagePack writes: a private or [IgnoreMember] one leaves the type integer-keyed and owned.
[Test]
public void AStringKeyOnAMemberMessagePackDoesNotWriteLeavesTheTypeOwned()
{
foreach (var holder in new object[] { new PSkH { Items = { new EIded(), new EIded() } }, new ISkH { Items = { new EIded(), new EIded() } } })
{
Assert.That(Problems(holder.GetType()), Is.Empty, holder.GetType().Name);
Assert.That(CultElementIds.Inspect(holder.GetType()).IdPath, Does.Contain(holder.GetType()), holder.GetType().Name);
Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name);
Assert.That(CultElementIds.Holds(holder.GetType(), holder), Is.True, holder.GetType().Name);
}
Assert.That(CultElementIds.WalksAsOwner(typeof(SkOver)), Is.False, "a string key inherited by an override still makes the type one whole value");
Assert.That(CultElementIds.WalksAsOwner(typeof(PSkH)), Is.True);
Assert.That(Problems(typeof(Whole<List<NoIdPlain>>)).Single(), Does.Contain("[CultElementId]"));
Assert.That(Problems(typeof(Whole<List<NoIdEl>>)).Single(), Does.Contain("[CultElementId]"), "an element whose only string key is ignored is as bare as its twin");
}
// The AllowPrivate edges: an [IgnoreMember] member is not written even under it, and only the element's own most derived
// declaration lets a private setter be read back.
[Test]
public void TheAllowPrivateEdgesFollowWhatMessagePackWrites()
{
Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApIgnAp)), Is.Empty, "an ignored member is not written under AllowPrivate");
Assert.That(Problems(typeof(HApIgnId)).Single(), Does.Contain("does not write").And.Contain("Id"));
Assert.That(Problems(typeof(HNoApPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id"),
"the derived plain [MessagePackObject] decides: its private setter is not read back, whatever its base declares");
}
// MessagePack writes an AllowPrivate type with every member up its base chain, and the most derived [MessagePackObject]
// declaration decides whether that is so.
[Test]
public void AnAllowPrivateTypeIsWalkedByTheMembersMessagePackWrites()
{
Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" }));
Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" }));
Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApInheritor)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" }),
"AllowPrivate is inherited by a subclass that declares no attribute of its own");
var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;
var held = new HOptHidden();
held.Peek.Add(new EIded());
Assert.That(Assign(held, "r", deterministic: false), Is.EqualTo(1));
var back = MessagePackSerializer.Deserialize<HOptHidden>(MessagePackSerializer.Serialize(held, options), options);
Assert.That(back.Peek.Single().Id, Is.EqualTo(held.Peek.Single().Id), "the minted id of a private list survives a round trip");
Assert.That(Problems(typeof(HApPrivSet)), Is.Empty, "MessagePack reads an AllowPrivate type's private setter back");
var privSet = new HApPrivSet { Items = { new EApPrivSet(), new EApPrivSet() } };
Assert.That(Assign(privSet, "r", deterministic: false), Is.EqualTo(2));
var round = MessagePackSerializer.Deserialize<HApPrivSet>(MessagePackSerializer.Serialize(privSet, options), options);
Assert.That(round.Items.Select(item => item.Id), Is.EqualTo(privSet.Items.Select(item => item.Id)));
Assert.That(round.Items.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$"));
Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public"), "without AllowPrivate it is still refused");
}
private static string?[] Ids(object holder) => holder switch
{
HInit init => init.Items.Select(item => item.Id).ToArray(),
HCtorId ctor => ctor.Items.Select(item => item.Id).ToArray(),
_ => throw new ArgumentException(holder.GetType().Name),
};
// A subclass that redeclares the id member does not change which member the declared element writes.
[Test]
public void AnElementIsReadByItsDeclaredTypeNotItsRuntimeType()
{
Assert.That(Problems(typeof(HDoc2)), Is.Empty);
static HE Hd(string? declared = null, string? redeclared = null)
{
var item = new HD { Id = redeclared };
((HE)item).Id = declared;
return item;
}
var minted = new HDoc2 { Items = { Hd() } };
Assert.That(Assign(minted, "r", deterministic: false), Is.EqualTo(1));
Assert.That(minted.Items[0].Id, Does.Match("^[0-9a-f]{12}$"));
Assert.That(((HD)minted.Items[0]).Id, Is.Null, "the redeclared member is never written, so it is not minted");
Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "not-hex") } }, "r", false),
Throws.TypeOf<CultElementIdException>(), "the declared member is checked");
Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "not-hex") } }, "r", false),
Throws.Nothing, "the redeclared member is never read");
Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "0123456789ab") } }), Is.True);
Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "0123456789ab") } }), Is.False);
}
// ---- shapes ----
[MessagePackObject] public class PubIgn { [Key(0)] [IgnoreMember] public List<EBare> L { get; set; } = new(); }
#pragma warning disable CS0169, CS0414, CS0649
[MessagePackObject] public class ApIgn { [Key(0)] [IgnoreMember] private List<EIded> l = new(); }
#pragma warning restore CS0169, CS0414, CS0649
[MessagePackObject] public class PgHold { [Key(0)] public List<EIded> Items { private get; set; } = new(); }
[MessagePackObject]
public class MemberMix
{
[Key(0)] public int A;
[Key(1)] [IgnoreMember] public List<EBare>? Ign;
[Key(2)] public string? PS { get; private set; }
[Key(3)] public string? GetOnly { get; }
#pragma warning disable CS0169, CS0414, CS0649
[Key(4)] private int priv;
#pragma warning restore CS0169, CS0414, CS0649
[Key(5)] public string? InitOnly { get; init; }
[IgnoreMember] public int Priv => priv;
}
[MessagePackObject] public class EPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } }
[MessagePackObject] public class HPrivSet { [Key(0)] public List<EPrivSet> Items { get; set; } = new(); }
[MessagePackObject] public class EIgnId { [Key(0)] [IgnoreMember] [CultElementId] public string? Id { get; set; } }
[MessagePackObject] public class HIgnId { [Key(0)] public List<EIgnId> Items { get; set; } = new(); }
#pragma warning disable CS0169, CS0649
[MessagePackObject] public class EPrivateId { [Key(0)] [CultElementId] private string? id; }
#pragma warning restore CS0169, CS0649
[MessagePackObject] public class HPrivateId { [Key(0)] public List<EPrivateId> Items { get; set; } = new(); }
[MessagePackObject] public class EInit { [Key(0)] [CultElementId] public string? Id { get; init; } }
[MessagePackObject] public class HInit { [Key(0)] public List<EInit> Items { get; set; } = new(); }
[MessagePackObject]
public class ECtorId
{
[SerializationConstructor] public ECtorId(string? id) { Id = id; }
[Key(0)] [CultElementId] public string? Id { get; set; }
}
[MessagePackObject] public class HCtorId { [Key(0)] public List<ECtorId> Items { get; set; } = new(); }
[MessagePackObject] public class HE { [Key(0)] [CultElementId] public string? Id { get; set; } }
[MessagePackObject] public class HD : HE { [Key(2)] [CultElementId] public new string? Id { get; set; } }
[MessagePackObject] public class HDoc2 { [Key(0)] public List<HE> Items { get; set; } = new(); }
[MessagePackObject]
public class KNode : IEnumerable<KNode>
{
[Key(0)] [CultElementId] public string? Id { get; set; }
[Key(1)] public List<KNode> Children { get; set; } = new();
public IEnumerator<KNode> GetEnumerator() => Children.GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject] public class HKNode { [Key(0)] public List<KNode> Nodes { get; set; } = new(); }
public class UNode : IEnumerable<UNode>
{
public IEnumerator<UNode> GetEnumerator() { yield break; }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject] public class HUNode { [Key(0)] public UNode? N { get; set; } }
[MessagePackObject]
public class KBag : IEnumerable<EIded>
{
[Key(0)] public List<EIded> Items { get; set; } = new();
[Key(1)] public List<EBare> Other { get; set; } = new();
public IEnumerator<EIded> GetEnumerator() { yield break; }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject]
public class KBagIded : IEnumerable<EIded>
{
[Key(0)] public List<EIded> Items { get; set; } = new();
public IEnumerator<EIded> GetEnumerator() { yield break; }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; }
[MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; }
[MessagePackObject] public class ENest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List<EIded> Sub { get; set; } = new(); }
[MessagePackObject] public class EBareNest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List<EBare> Sub { get; set; } = new(); }
public sealed class NoFormatter { }
public class UnkeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; }
[Union(0, typeof(KeyedArm)), Union(1, typeof(UnkeyedArm))] public abstract class ArmBase { }
[MessagePackObject] public class KeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; }
[MessagePackObject] public class HUnkeyedArmList { [Key(0)] public List<ArmBase> Items { get; set; } = new(); }
[MessagePackObject] public class HUnkeyedArmField { [Key(0)] public ArmBase? Root { get; set; } }
[MessagePackObject]
public class HCollections
{
[Key(0)] public List<EIded> A { get; set; } = new();
[Key(1)] public EIded[] B { get; set; } = Array.Empty<EIded>();
[Key(2)] public Dictionary<string, HHasIds> C { get; set; } = new();
[Key(3)] public List<List<EIded>> D { get; set; } = new();
}
[MessagePackObject]
public class HValueListsBesideObjects
{
[Key(0)] public List<EIded> Objects { get; set; } = new();
[Key(1)] public List<int> Numbers { get; set; } = new();
[Key(2)] public List<PlainValue> Plain { get; set; } = new();
[Key(3)] public List<Pair> Points { get; set; } = new();
}
[MessagePackFormatter(typeof(NoFormatter))] public class Roles : List<EIded> { }
[MessagePackObject] public class HRoles { [Key(0)] public Roles Items { get; set; } = new(); }
[MessagePackFormatter(typeof(NoFormatter))] public class PlainValues : List<PlainValue> { }
[MessagePackObject] public class HFormattedValueCollection { [Key(0)] public List<EIded> Objects { get; set; } = new(); [Key(1)] public PlainValues Plain { get; set; } = new(); }
[MessagePackObject]
public abstract class FormattedIdBase
{
[Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual string Id { get; set; } = "";
}
[MessagePackObject]
public class FormattedIdOverride : FormattedIdBase
{
[Key(0)] [CultElementId] public override string Id { get; set; } = "";
}
[MessagePackObject] public class HOverFormattedId { [Key(0)] public List<FormattedIdOverride> Items { get; set; } = new(); }
[MessagePackObject]
public class OwnerBaseFormatted
{
[Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual List<EIded> Items { get; set; } = new();
}
[MessagePackObject]
public class OwnerOverFormatted : OwnerBaseFormatted
{
[Key(0)] public override List<EIded> Items { get; set; } = new();
}
[MessagePackObject]
public class IdVirtual
{
[Key(0)] public string X { get; set; } = "";
[Key(1)] [CultElementId] public virtual string Id { get; set; } = "";
}
[MessagePackObject] public class IdOverride : IdVirtual { [Key(1)] public override string Id { get; set; } = ""; }
[MessagePackObject] public class HIdOverride { [Key(0)] public List<IdOverride> Items { get; set; } = new(); }
[MessagePackObject] public class HUnkeyedTwoLists { [Key(0)] public List<ArmBase> A { get; set; } = new(); [Key(1)] public List<ArmBase> B { get; set; } = new(); }
public class PlainValue { public string X { get; set; } = ""; }
[MessagePackObject] public class HValueList { [Key(0)] public List<PlainValue> Items { get; set; } = new(); }
[MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedElement { [Key(0)] [CultElementId] public string Id { get; set; } = ""; }
[MessagePackObject] public class HFormattedElement { [Key(0)] public List<FormattedElement> Items { get; set; } = new(); }
[MessagePackObject] public class HFormattedList { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public List<EIded> Items { get; set; } = new(); }
[MessagePackObject] public class FormattedIdElement { [Key(0)] [CultElementId] [MessagePackFormatter(typeof(NoFormatter))] public string Id { get; set; } = ""; }
[MessagePackObject] public class HFormattedId { [Key(0)] public List<FormattedIdElement> Items { get; set; } = new(); }
[MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedOwner { [Key(0)] public List<EIded> Items { get; set; } = new(); }
[MessagePackObject] public class HFormattedOwner { [Key(0)] public FormattedOwner Owner { get; set; } = new(); }
[MessagePackObject] public class HFormattedMember { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public HHasIds Inner { get; set; } = new(); }
[MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedLeaf { [Key(0)] public int N { get; set; } }
[MessagePackObject] public class HFormattedField { [Key(0)] public FormattedLeaf Leaf { get; set; } = new(); [Key(1)] public List<string> Tags { get; set; } = new(); }
[MessagePackObject] public class FormattedDerived : FormattedElement { }
[MessagePackObject] public class HFormattedDerived { [Key(0)] public List<FormattedDerived> Items { get; set; } = new(); }
[Union(0, typeof(IdArm))] public abstract class IdBase { }
[MessagePackObject] public class IdArm : IdBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List<EIded> Items { get; set; } = new(); }
[Union(0, typeof(PlainArm))] public abstract class PlainBase { }
[MessagePackObject] public class PlainArm : PlainBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; }
[MessagePackObject] public class HPlainBaseList { [Key(0)] public List<PlainBase> Items { get; set; } = new(); }
[MessagePackObject] public class HIdBaseOwner { [Key(0)] public IdBase? Root { get; set; } }
[MessagePackObject] public class HLeft { [Key(0)] public HCycle Cycle { get; set; } = new(); }
[MessagePackObject] public class HRight { [Key(0)] public HCycle Cycle { get; set; } = new(); }
[MessagePackObject] public class HCycle { [Key(0)] public HCycleBack Back { get; set; } = new(); }
[MessagePackObject] public class HCycleBack { [Key(0)] public HCycle? Up { get; set; } [Key(1)] public List<EIded> Items { get; set; } = new(); }
[MessagePackObject] public class HDiamond { [Key(0)] public HLeft Left { get; set; } = new(); [Key(1)] public HRight Right { get; set; } = new(); }
[MessagePackObject] public class HList { [Key(0)] public List<EBare> Items { get; set; } = new(); }
[MessagePackObject] public class HArray { [Key(0)] public EBare[] Items { get; set; } = Array.Empty<EBare>(); }
[MessagePackObject] public class HDictionary { [Key(0)] public Dictionary<string, HList> Map { get; set; } = new(); }
@ -337,6 +965,80 @@ namespace GameCult.Caching.Tests
[Key(2)] public List<ENest> Nest { get; set; } = new();
}
[MessagePackObject] public struct SHold { [Key(0)] public List<EBare> Items; }
[MessagePackObject] public struct SHoldIded { [Key(0)] public List<EIded> Items; }
[MessagePackObject] public struct SValue { [Key(0)] public int N; }
public struct UnkeyedS { [Key(0)] public List<EBare> Items; }
[MessagePackObject] public class HUnkeyedStruct { [Key(0)] public UnkeyedS Value; }
public class InheritedBare : EBare { }
[MessagePackObject] public class HInheritedBare { [Key(0)] public List<InheritedBare> Items { get; set; } = new(); }
[MessagePackObject] public class HNullableStructIded { [Key(0)] public SHoldIded? Hold; }
[MessagePackObject] public class HStruct { [Key(0)] public SHold Hold; }
[MessagePackObject] public class HStructList { [Key(0)] public List<SHold> Holds { get; set; } = new(); }
[MessagePackObject] public class HNullableStruct { [Key(0)] public SHold? Hold; }
[MessagePackObject] public class HStructIded { [Key(0)] public SHoldIded Hold; [Key(1)] public List<SHoldIded> Many = new(); }
[MessagePackObject] public class HStructValues { [Key(0)] public List<SValue> Structs = new(); [Key(1)] public List<EIded> Objects = new(); }
[MessagePackObject] public class HReadOnlyMap { [Key(0)] public IReadOnlyDictionary<string, HList> Map { get; set; } = new Dictionary<string, HList>(); }
[MessagePackObject] public class HReadOnlyIded { [Key(0)] public IReadOnlyDictionary<string, HHasIds> Map { get; set; } = new Dictionary<string, HHasIds>(); }
[MessagePackObject] public class HMapIded { [Key(0)] public Dictionary<string, HHasIds> Map { get; set; } = new(); }
public sealed class RoMap : IReadOnlyDictionary<string, HHasIds>
{
private readonly Dictionary<string, HHasIds> _inner;
public RoMap(Dictionary<string, HHasIds> inner) => _inner = inner;
public HHasIds this[string key] => _inner[key];
public IEnumerable<string> Keys => _inner.Keys;
public IEnumerable<HHasIds> Values => _inner.Values;
public int Count => _inner.Count;
public bool ContainsKey(string key) => _inner.ContainsKey(key);
public bool TryGetValue(string key, out HHasIds value) => _inner.TryGetValue(key, out value!);
public IEnumerator<KeyValuePair<string, HHasIds>> GetEnumerator() => _inner.GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject] public class BaseC { [Key(0)] public int X { get; set; } }
[MessagePackObject] public class DerC : BaseC { [Key(1)] public List<EIded> L { get; set; } = new(); }
[MessagePackObject] public class HBase { [Key(0)] public BaseC B { get; set; } = new(); }
[MessagePackObject] public class HObject { [Key(0)] public object? O { get; set; } }
#pragma warning disable CS0169, CS0414, CS0649
[MessagePackObject] public class HHiddenList { [Key(0)] private List<EIded> Stash { get; set; } = new(); }
[MessagePackObject] public class HHiddenOwner { [Key(0)] private HHasIds stash = new(); }
[MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List<EIded> stash = new(); [IgnoreMember] public List<EIded> Peek => stash; }
[MessagePackObject] public class HHiddenBase { [Key(0)] private List<EIded> stash = new(); }
[MessagePackObject] public class HHiddenFromBase : HHiddenBase { [Key(1)] public int N { get; set; } }
[MessagePackObject] public class HHiddenLeaf { [Key(0)] public List<EIded> Items { get; set; } = new(); [Key(1)] private int secret = 7; }
[MessagePackObject] public class ApBase { [Key(0)] private List<EIded> baseStash = new(); }
[MessagePackObject(AllowPrivate = true)] public class ApDerived : ApBase { [Key(1)] private List<EIded> own = new(); [Key(2)] public int N { get; set; } [Key(3)] public List<EIded> Pg { private get; set; } = new(); }
[MessagePackObject(AllowPrivate = true)] public class ApBase2 { [Key(0)] private List<EIded> baseStash = new(); }
[MessagePackObject] public class NoApDerived : ApBase2 { [Key(1)] public int N { get; set; } }
#pragma warning restore CS0169, CS0414, CS0649
[MessagePackObject] public class Whole<T> { [Key(0)] public T? Value { get; set; } }
public class NkHold { public List<EIded> Items = new(); }
[MessagePackObject] public class SkHold { [Key("items")] public List<EIded> Items = new(); }
[MessagePackObject(keyAsPropertyName: true)] public class KpBase { [Key(0)] public List<EIded> Items = new(); }
public class KpDerived : KpBase { }
[MessagePackObject] public class MpBase { [Key(0)] public List<EIded> Items = new(); }
[System.Runtime.Serialization.DataContract] public class DcOverMp : MpBase { }
public class ApInheritor : ApDerived { }
[MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; }
[System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List<EIded> Items = new(); }
[MessagePackObject(AllowPrivate = true)] public class ApIgnAp { [Key(0)] [IgnoreMember] private List<EIded> l = new(); }
[MessagePackObject] public class PSkH { [Key(0)] public List<EIded> Items = new(); [Key("x")] private int x = 1; }
[MessagePackObject] public class ISkH { [Key(0)] public List<EIded> Items = new(); [IgnoreMember] [Key("x")] public int X { get; set; } }
[MessagePackObject] public class SkOverBase { [Key("items")] public virtual List<EIded> Items { get; set; } = new(); }
[MessagePackObject] public class SkOver : SkOverBase { public override List<EIded> Items { get; set; } = new(); }
[MessagePackObject] public class NoIdEl { [Key(0)] public int N { get; set; } [IgnoreMember] [Key("x")] public int X { get; set; } }
[MessagePackObject] public class NoIdPlain { [Key(0)] public int N { get; set; } }
[MessagePackObject(AllowPrivate = true)] public class EApIgnId { [Key(0)] public int N { get; set; } [Key(1)] [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; }
[MessagePackObject] public class HApIgnId { [Key(0)] public List<EApIgnId> Items { get; set; } = new(); }
[MessagePackObject(AllowPrivate = true)] public class ApIdBase { [Key(0)] public int N { get; set; } }
[MessagePackObject] public class ENoApPrivSet : ApIdBase { [Key(1)] [CultElementId] public string? Id { get; private set; } }
[MessagePackObject] public class HNoApPrivSet { [Key(0)] public List<ENoApPrivSet> Items { get; set; } = new(); }
[MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } }
[MessagePackObject] public class HApPrivSet { [Key(0)] public List<EApPrivSet> Items { get; set; } = new(); }
[MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; }
[MessagePackObject] public class HDerivedText { [Key(0)] public List<EDerivedText> Items { get; set; } = new(); }

123
tests/GameCult.Caching.Tests/ElementIdTests.cs
View file

@ -1,6 +1,7 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reflection;
@ -356,6 +357,48 @@ namespace GameCult.Caching.Tests
private static byte[] OldShapedReels(string label) =>
MessagePackSerializer.Serialize(new List<OldReel> { new() { Label = label, Marks = { new OldMark { Text = "x" }, new OldMark { Text = "y" } } } });
// A holder the walk does not own (ImmutableArray here) is one whole value: its elements carry no element ids, so it never
// marks the store, and a variant replaces it whole (q3 amended 2026-10-16).
[Test]
public void AWholeListHolderNeverMarksTheStoreAndAVariantReplacesItWhole()
{
Assert.That(CultElementIds.CanHoldIds(typeof(IdWholeHolder)), Is.False);
var path = PathOf("whole-holder.cc");
using (var cache = OpenWith(path, false, typeof(IdWholeHolder)))
{
cache.Commit(batch => batch.Upsert(typeof(IdWholeHolder),
new IdWholeHolder { Name = "base", Reels = ImmutableArray.Create(new IdReel { Label = "set", Id = HexA }, new IdReel { Label = "unset" }) },
new CultRecordKey("base")));
}
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "ids inside a whole value mark nothing");
using (var fresh = OpenWith(path, false, typeof(IdWholeHolder)))
{
fresh.Commit(batch => batch.Upsert(typeof(IdWholeHolder), new IdWholeHolder { Name = "other" }, new CultRecordKey("other")));
var back = fresh.Get<IdWholeHolder>(new CultRecordKey("base"))!;
Assert.That(back.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { HexA, "" }), "set ids are plain values, unset ids stay unset");
}
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"));
using (var cache = OpenWith(path, false, typeof(IdWholeHolder)))
{
var replacement = ImmutableArray.Create(new IdReel { Label = "v", Id = "eeeeeeeeeeee" });
cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"),
new[] { cache.Override<IdWholeHolder>(nameof(IdWholeHolder.Name), "v"), cache.Override<IdWholeHolder>(nameof(IdWholeHolder.Reels), replacement) }));
Assert.That(cache.Get<IdWholeHolder>(new CultRecordKey("v"))!.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { "eeeeeeeeeeee" }), "replaced whole");
Assert.That(cache.Get<IdWholeHolder>(new CultRecordKey("base"))!.Reels, Has.Length.EqualTo(2), "the base is untouched");
}
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "a variant makes v2, and nothing in it holds element ids");
using var again = OpenWith(path, false, typeof(IdWholeHolder));
Assert.That(again.Get<IdWholeHolder>(new CultRecordKey("v"))!.Reels.Single().Id, Is.EqualTo("eeeeeeeeeeee"));
var step = new CultVariantOverride(CultOverrideOp.Set, new[] { new CultPathStep(1, HexA) }, string.Empty, MessagePackSerializer.Serialize("x"));
var refused = Assert.Catch<Exception>(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { again.Override<IdWholeHolder>(nameof(IdWholeHolder.Name), "p"), step })))!;
Assert.That(refused.ToString(), Does.Contain("top-level Set overrides only"));
}
[Test]
public void AStoreThatCanHoldElementIdsCarriesTheV3MarkerAndOneThatCannotStaysV1()
{
@ -1020,6 +1063,78 @@ namespace GameCult.Caching.Tests
Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1"));
}
// Commits one plain item through a cache over `path` and reports how many copied records that write decoded to ask whether
// they hold an element id, and the header it left.
private (int Decoded, string Header) CommitItemCountingDecodes(string path, params Type[] types)
{
var registry = CultDocumentRegistry.ForTypes(types);
var store = new SingleFileMessagePackBackingStore(path);
using var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
cache.AddBackingStore(store);
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
store.CopiedRecordsDecoded = 0;
cache.Commit(batch => batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, new CultRecordKey("p")));
return (store.CopiedRecordsDecoded, HeaderOf(path));
}
// The copy check decodes a record only where the answer can change: each gate, removed, makes one of these counts wrong.
[Test]
public void TheCopyCheckDecodesOnlyWhatCouldMarkTheStore()
{
// A v3 header is already marked: nothing to decide.
var marked = PathOf("gate-marked.cc");
WritePreIdStore(marked, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1));
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(marked));
snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3;
File.WriteAllBytes(marked, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
Assert.That(CommitItemCountingDecodes(marked, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v3")), "a marked store decodes nothing");
// Records whose type reaches no object list can hold no id.
var noLists = PathOf("gate-no-lists.cc");
using (var seed = OpenWith(noLists, false, typeof(PreCut2FixtureItem)))
{
foreach (var key in new[] { "x", "y", "z" })
UpsertPlain(seed, key);
}
Assert.That(CommitItemCountingDecodes(noLists, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a type with no object list is not decoded");
// A written record that holds an id already marks the store, whichever of the written records it is.
var stagedPath = PathOf("gate-staged.cc");
WritePreIdStore(stagedPath, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1));
var stagedStore = new SingleFileMessagePackBackingStore(stagedPath);
var stagedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck), typeof(PreCut2FixtureItem) });
using (var cache = new CultCache(stagedRegistry, CultCacheMessagePack.CreateCodec(stagedRegistry)))
{
cache.AddBackingStore(stagedStore);
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
stagedStore.CopiedRecordsDecoded = 0;
cache.Commit(batch =>
{
batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain"));
batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh"));
});
}
Assert.That(stagedStore.CopiedRecordsDecoded, Is.EqualTo(0), "a staged record holding an id marks without asking");
Assert.That(HeaderOf(stagedPath), Is.EqualTo("cultcache.store.v3"));
// Id-capable records without ids: each is decoded once, and the store stays unmarked.
var capable = PathOf("gate-capable.cc");
WritePreIdStore(capable, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1));
Assert.That(CommitItemCountingDecodes(capable, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((3, "cultcache.store.v1")));
// A record the writer's registry does not claim is foreign: no type, no decode.
var foreign = PathOf("gate-foreign.cc");
WritePreIdStore(foreign, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1));
Assert.That(CommitItemCountingDecodes(foreign, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a foreign record is not decoded");
// A variant is copied as its bytes: only a C# write of one holding ids marks the store, and that write marked it already.
var variant = PathOf("gate-variant.cc");
WriteV2StoreWithAnIdLessVariant(variant);
Assert.That(CommitItemCountingDecodes(variant, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((1, "cultcache.store.v2")), "the plain base is decoded, the variant is not");
}
// Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta
// it read, so the file holds none and no write of a plain record beside it may mark the file.
private static void WriteV2StoreWithAnIdLessVariant(string path)
@ -1215,6 +1330,14 @@ namespace GameCult.Caching.Tests
[Key(1)] public int Depth { get; set; }
}
[CultDocument("tests.element_id_whole_holder", "tests.element_id_whole_holder.v1")]
[MessagePackObject]
public sealed class IdWholeHolder
{
[Key(0)] [CultName] public string Name { get; set; } = "";
[Key(1)] public ImmutableArray<IdReel> Reels { get; set; } = ImmutableArray<IdReel>.Empty;
}
[CultDocument("tests.element_id_deck", "tests.element_id_deck.v1")]
[MessagePackObject]
public sealed class IdDeck

175
tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs
View file

@ -67,6 +67,181 @@ namespace GameCult.Caching.Tests
}
}
private static string HeaderOf(string path) =>
CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion;
private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledAsync(string path)
{
var cache = new CultCache();
var store = new SingleFileMessagePackBackingStore(path);
cache.AddBackingStore(store);
await cache.PullAllBackingStoresAsync();
return (cache, store);
}
// cultcache-py writes element ids under an unmarked v1 header. A C# write that carries such a record forward marks the
// store, because a reader from before ids would otherwise rewrite the elements without them.
[Test]
public async Task ACSharpWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore()
{
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Store("leaf-record.cc", scratch);
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "cultcache-py leaves the header unmarked");
var (cache, store) = await PulledAsync(path);
await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" });
store.PushAll();
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3));
var (reloaded, _) = await PulledAsync(path);
var py = reloaded.AllEntries.OfType<NumericLeafVectorTests.LeafRecord>().Single(record => record.Id == "py");
Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" }));
}
finally
{
Directory.Delete(scratch, true);
}
}
// What a load saw is a fact about those bytes at that version. Another writer replaced the record since, so the copy this
// write makes is not the record the load saw, and the header is not judged by it.
[Test]
public async Task ACopiedRecordAnotherWriterChangedIsNotJudgedByTheOldRead()
{
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Store("leaf-record.cc", scratch);
var (reader, readerStore) = await PulledAsync(path);
// Another writer replaces the record with one that holds no id.
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var replaced = snapshot.Records.Single(record => record.Key == "py");
replaced.Payload = CultDocumentMessagePackSerialization.SerializeUntyped(
new NumericLeafVectorTests.LeafRecord { Id = "py" }, typeof(NumericLeafVectorTests.LeafRecord));
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the replacement holds no id");
await reader.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" });
readerStore.PushAll();
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1));
}
finally
{
Directory.Delete(scratch, true);
}
}
// A v1 store holding only this record c0, opened by a cache that has pulled it; then another runtime appends the python
// record (and its payload rewritten by `change`) to the file raw, header still v1, the way a writer that keeps the v1 it
// found does. Returns the open cache and its store.
private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledThenAppended(
string path, string pySource, Action<CultPersistedRecord>? change = null)
{
using (var seed = new CultCache())
{
var seedStore = new SingleFileMessagePackBackingStore(path);
seed.AddBackingStore(seedStore);
await seed.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c0" });
seedStore.PushAll();
}
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1));
var opened = await PulledAsync(path);
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var pySnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource));
var py = pySnapshot.Records.Single(record => record.Key == "py");
change?.Invoke(py);
snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray();
snapshot.SchemaCatalog = snapshot.SchemaCatalog.Concat(pySnapshot.SchemaCatalog).GroupBy(entry => entry.SchemaId).Select(group => group.First()).ToArray();
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found");
return opened;
}
// The cache never read this record: it was appended after the pull. A per-instance memory of what the pull saw cannot
// know about it; the write decides from the bytes the file holds under its lock.
[Test]
public async Task ARecordAForeignWriterAddsAfterThePullMarksTheStoreWhenCopied()
{
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Path.Combine(scratch, "after-pull.cc");
var (cache, store) = await PulledThenAppended(path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc")));
await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" });
store.PushAll();
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3));
var (reloaded, _) = await PulledAsync(path);
var py = reloaded.AllEntries.OfType<NumericLeafVectorTests.LeafRecord>().Single(record => record.Id == "py");
Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" }));
}
finally
{
Directory.Delete(scratch, true);
}
}
// Ids a load minted in memory are not on disk: the check judges the file's bytes, never the document the cache holds.
[Test]
public async Task ARecordWhoseIdsExistOnlyInMemoryLeavesTheStoreUnmarked()
{
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Path.Combine(scratch, "in-memory.cc");
var source = Store("leaf-record.cc", scratch);
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(source));
var idless = new NumericLeafVectorTests.LeafRecord { Id = "py" };
idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "one" });
idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "two" });
snapshot.Records.Single().Payload = CultDocumentMessagePackSerialization.SerializeUntyped(idless, typeof(NumericLeafVectorTests.LeafRecord));
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single().Payload;
var (cache, store) = await PulledAsync(path);
Assert.That(cache.AllEntries.OfType<NumericLeafVectorTests.LeafRecord>().Single().Items.Select(item => item.Id), Has.All.Length.EqualTo(12), "the load minted ids in memory");
await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" });
store.PushAll();
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1));
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py").Payload;
Assert.That(after, Is.EqualTo(before), "the copied record is the bytes the file held");
}
finally
{
Directory.Delete(scratch, true);
}
}
// The write-set rule copies records without interpreting them, so the check must never refuse a write: a record this
// cache claims but cannot decode is copied as its bytes and decides nothing.
[Test]
public async Task ACopiedRecordThatDoesNotDecodeIsCopiedAndDecidesNothing()
{
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Path.Combine(scratch, "undecodable.cc");
var garbage = new byte[] { 0xc1, 0xc1, 0xc1 };
var (cache, store) = await PulledThenAppended(
path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc")), record => record.Payload = garbage);
await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" });
Assert.DoesNotThrow(() => store.PushAll());
Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1));
var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py");
Assert.That(py.Payload, Is.EqualTo(garbage));
}
finally
{
Directory.Delete(scratch, true);
}
}
[TestCase(3)]
[TestCase(5)]
[TestCase(7)]

557
tests/GameCult.Caching.Tests/UnionArmGuardTests.cs
View file

@ -4,6 +4,7 @@ using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
@ -20,7 +21,7 @@ namespace GameCult.Caching.Tests
{
private const string Canary = "CANARY-9d1e47";
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc) });
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc), typeof(StubDoc) });
private delegate void Emit(ref MessagePackWriter writer);
@ -246,15 +247,543 @@ namespace GameCult.Caching.Tests
[Test]
public void ConsumerResolverStillEncodesArms()
{
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new GuardDoc { Id = "d1", Root = new StubArm() }, typeof(GuardDoc), Registry);
Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Doc((ref MessagePackWriter w) =>
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new StubDoc { Id = "d1", Root = new StubArm() }, typeof(StubDoc), Registry);
Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Raw((ref MessagePackWriter w) =>
{
w.WriteArrayHeader(2); w.Write("d1");
w.WriteArrayHeader(2); w.Write(9); w.WriteArrayHeader(1); w.Write("via-resolver");
}))));
var back = (GuardDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(GuardDoc), bytes, Registry);
var back = (StubDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(StubDoc), bytes, Registry);
Assert.That(back.Root, Is.TypeOf<StubArm>());
}
// A list element is written by its [Key] layout or its id never reaches disk: a consumer resolver that claims the arm of
// a list's union is refused on every path that encodes or decodes the document, naming the arm and the resolver, and
// writes nothing.
[Test]
public void ListElementsClaimedByAResolverAreRefused()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ClaimDoc) });
var doc = new ClaimDoc { Id = "c1", Items = { new ClaimArm { N = 4 } } };
void Refused(TestDelegate attempt)
{
var thrown = Assert.Throws<InvalidOperationException>(attempt)!;
Assert.That(thrown.Message, Does.Contain("ClaimArm").And.Contain(nameof(ClaimArmResolver)).And.Contain(nameof(ClaimDoc)));
}
Refused(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ClaimDoc), registry));
var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("c1"); w.WriteArrayHeader(0); });
Refused(() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ClaimDoc), held, registry));
Refused(() => CultCacheMessagePack.CreateCodec(registry).Serialize(doc.Items, typeof(List<ClaimNode>), typeof(ClaimDoc)));
var scratch = Directory.CreateTempSubdirectory().FullName;
try
{
var path = Path.Combine(scratch, "claim.cc");
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
Assert.That(Assert.Catch(() => cache.Commit(batch => batch.Upsert(typeof(ClaimDoc), doc, new CultRecordKey("c1"))))!.ToString(),
Does.Contain(nameof(ClaimArmResolver)));
Assert.That(File.Exists(path), Is.False, "nothing reached disk");
}
finally
{
Directory.Delete(scratch, true);
}
}
// A resolver's formatter for the list type of an object list drops the elements' ids as surely as one for the element,
// so the collection type is on the id path (Soul s1 P1). A claim on a value list, even beside an object list, is none.
[Test]
public void AResolverClaimingTheListTypeOfAnObjectListIsRefused()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(LcDoc), typeof(LcValueDoc) });
var doc = new LcDoc { Id = "l1", Items = { new LcArm { Id = "000000000001", N = 4 } } };
var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("l1"); w.WriteArrayHeader(0); });
foreach (var attempt in new TestDelegate[]
{
() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(LcDoc), registry),
() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcDoc), held, registry)
})
{
var thrown = Assert.Throws<InvalidOperationException>(attempt)!;
Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain("List`1").And.Contain(nameof(LcDoc)));
}
var values = new LcValueDoc
{
Id = "v1",
Items = { new LcOther { Id = "000000000002" } },
Values = { new LcValue { N = 3 }, new LcValue { N = 5 } },
Points = { new Pair { A = 1, B = 2 } }
};
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(values, typeof(LcValueDoc), registry);
var back = (LcValueDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcValueDoc), bytes, registry);
Assert.That(back.Items.Single().Id, Is.EqualTo("000000000002"));
Assert.That(back.Values.Select(value => value.N), Is.EqualTo(new[] { 3, 5 }));
Assert.That(back.Points.Single(), Is.EqualTo(new Pair { A = 1, B = 2 }));
}
// A resolver that hands out MessagePack's own formatter for a type writes its [Key] layout: it claims nothing, however it
// got there, and the document encodes with the id present (Soul s1 P2).
[Test]
public void AResolverHandingBackTheStandardFormatterIsNoClaim()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(FbDoc) });
var doc = new FbDoc { Id = "f1", Items = { new FbArm { Id = "000000000009", N = 7 } } };
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(FbDoc), registry);
Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(doc, CultDocumentMessagePackSerialization.Options))));
var back = (FbDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(FbDoc), bytes, registry);
Assert.That(back.Items.Single().Id, Is.EqualTo("000000000009"));
}
// A consumer whose resolver composite ends in StandardResolverAllowPrivate (some models keep private [Key] members) gets
// that resolver's formatter for every keyed type. It writes the [Key] layout plus the private members, so it is no
// claim: the document encodes, the id is on disk, and the private value survives.
[Test]
public void AStandardResolverAllowPrivateFormatterIsNoClaim()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApDoc), typeof(ApPlainDoc) });
var doc = new ApDoc { Id = "a1", Items = { new ApArm { Id = "00000000000a", N = 4 }.WithSecret(11) } };
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApDoc), registry);
var back = (ApDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ApDoc), bytes, registry);
Assert.That(back.Items.Single().Id, Is.EqualTo("00000000000a"));
Assert.That(back.Items.Single().Secret, Is.EqualTo(11), "a private plain-value [Key] member is written");
var plain = new ApPlainDoc { Id = "a2", Items = { new ApPlainArm { Id = "00000000000b", N = 5 } } };
var plainBytes = CultDocumentMessagePackSerialization.SerializeUntyped(plain, typeof(ApPlainDoc), registry);
Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options))));
}
// The walk reads a [MessagePackObject] type by its keys, but MessagePack writes one that is also an ICollection<T> as a
// collection and drops the keys. The codec compares the two for every type on the id path and refuses the document.
[Test]
public void AKeyedTypeMessagePackWritesAsACollectionIsRefused()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(KCollDoc), typeof(KBagDoc) });
Assert.That(CultElementIds.Inspect(typeof(KCollDoc)).IdPath, Does.Contain(typeof(KColl)));
var coll = new KCollDoc { Id = "k1" };
coll.Items.Add(new FbArm { Id = "00000000000c" });
var thrown = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.SerializeUntyped(coll, typeof(KCollDoc), registry))!;
Assert.That(thrown.Message, Does.Contain($"writes {nameof(KColl)} as a collection, but the element-id walk reads it by its [Key] layout"));
// A keyed map is read by key too, so a keyed Dictionary subclass is on the path and refused the same way.
var map = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(KDictDoc)))!;
Assert.That(map.Message, Does.Contain($"writes {nameof(KDict)} as a collection"));
var bag = new KBagDoc { Id = "b1", Bag = new KBagE { Items = { new FbArm { Id = "00000000000d" } } } };
var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(bag, typeof(KBagDoc), registry);
var back = (KBagDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(KBagDoc), bytes, registry);
Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d"));
}
// Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused.
[Test]
public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter()
{
foreach (var holder in new[]
{
typeof(G<List<FbArm>>), typeof(G<FbArm[]>), typeof(G<FbArm[,]>), typeof(G<IList<FbArm>>), typeof(G<IReadOnlyList<FbArm>>),
typeof(G<ICollection<FbArm>>), typeof(G<IReadOnlyCollection<FbArm>>), typeof(G<IEnumerable<FbArm>>), typeof(G<HashSet<FbArm>>),
typeof(G<Queue<FbArm>>), typeof(G<Stack<FbArm>>), typeof(G<LinkedList<FbArm>>),
typeof(G<System.Collections.ObjectModel.ReadOnlyCollection<FbArm>>), typeof(G<System.Collections.ObjectModel.ObservableCollection<FbArm>>),
typeof(G<System.Collections.Immutable.ImmutableList<FbArm>>), typeof(G<ILookup<string, FbArm>>), typeof(G<List<List<FbArm>>>),
typeof(G<Dictionary<string, GHold>>), typeof(G<IDictionary<string, GHold>>), typeof(G<IReadOnlyDictionary<string, GHold>>),
typeof(G<SortedDictionary<string, GHold>>), typeof(G<System.Collections.Concurrent.ConcurrentDictionary<string, GHold>>),
typeof(G<System.Collections.ObjectModel.ReadOnlyDictionary<string, GHold>>), typeof(G<System.Collections.Immutable.ImmutableDictionary<string, GHold>>),
typeof(G<SortedList<string, GHold>>), typeof(G<FbArm>), typeof(G<IGu>), typeof(G<GStruct>),
})
{
if (holder != typeof(G<FbArm>))
Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.ToString());
Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString());
}
}
// One recogniser decides what the walk owns and the door only verifies it: the same types give no registration problem and a
// passing door. A type formatted by attribute or written by the object resolver is one whole value, off the id path.
[Test]
public void OneRecogniserDecidesAndTheDoorAgrees()
{
foreach (var holder in new[] { typeof(G<FormattedRoles>), typeof(G<ApTrueHold>), typeof(G<DcSeq>), typeof(G<DcColl>), typeof(G<DcBad>) })
{
Assert.That(CultElementIds.Inspect(holder).Problems, Is.Empty, holder.ToString());
Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString());
}
foreach (var whole in new[] { typeof(G<FormattedRoles>), typeof(G<DcSeq>), typeof(G<DcColl>), typeof(G<DcBad>) })
Assert.That(CultElementIds.Inspect(whole).IdPath, Is.Empty, whole.ToString());
var ap = CultElementIds.Inspect(typeof(G<ApTrueHold>)).IdPath;
Assert.That(ap, Does.Contain(typeof(ApTrueHold)).And.Contain(typeof(List<FbArm>)));
var seq = new G<DcSeq> { Value = new DcSeq { Items = { new FbArm() } } };
Assert.That(CultElementIds.Plan(seq.GetType(), seq, "r", deterministic: false).Count, Is.EqualTo(0));
}
// MessagePack throws for a type, or StandardResolver has no formatter for it (an interface or abstract class that is only
// IEnumerable<T>): such a type cannot be persisted, so the door refuses it with the same typed text as a throwing build.
[Test]
public void ATypeMessagePackCannotBuildIsATypedRefusal()
{
foreach (var (holder, name) in new[] { (typeof(G<IArms>), nameof(IArms)), (typeof(G<AbsArms>), nameof(AbsArms)) })
{
var thrown = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString())!;
Assert.That(thrown.Message, Does.Contain(name).And.Contain("cannot build a formatter"));
Assert.That(thrown.Message, Does.Contain("StandardResolver has no formatter for it"), "a null formatter says so, without a thrown message");
}
}
// A keyed type that enumerates lists is read by key and is on the id path, so the door checks it: MessagePack writes it as a
// collection, which loses the ids the walk would mint.
[Test]
public void AKeyedTypeThatEnumeratesListsIsCheckedByTheDoor()
{
Assert.That(CultElementIds.Inspect(typeof(G<KLL>)).IdPath, Does.Contain(typeof(KLL)));
var thrown = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G<KLL>)))!;
Assert.That(thrown.Message, Does.Contain($"writes {nameof(KLL)} as a collection"));
}
// A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter
// the claim check asks about (the claim stands) or a resolver ahead (it fails to build).
[Test]
public void AResolverThatThrowsIsATypedRefusal()
{
var first = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc)))!;
Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl)));
var second = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!;
Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build").And.Not.Contain("target of an invocation"),
"the resolver's own exception is named, not the reflection wrapper");
// A type MessagePack itself cannot build a formatter for (two public members on one key) is refused, naming the type.
var third = Assert.Throws<InvalidOperationException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc3)))!;
Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter").And.Not.Contain("target of an invocation"));
}
[MessagePackObject]
public sealed class KColl : List<FbArm>
{
[Key(0)] public int N;
}
[CultDocument("tests.kcoll_doc", "tests.kcoll_doc.v1")]
[MessagePackObject]
public sealed class KCollDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public KColl Items = new();
}
[MessagePackObject]
public sealed class KDict : Dictionary<string, GHold>
{
[Key(0)] public int N;
}
[CultDocument("tests.kdict_doc", "tests.kdict_doc.v1")]
[MessagePackObject]
public sealed class KDictDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public KDict Items = new();
}
[MessagePackObject]
public sealed class KBagE : IEnumerable<FbArm>
{
[Key(0)] public List<FbArm> Items = new();
public IEnumerator<FbArm> GetEnumerator() { yield break; }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[CultDocument("tests.kbag_doc", "tests.kbag_doc.v1")]
[MessagePackObject]
public sealed class KBagDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public KBagE Bag = new();
}
[MessagePackObject] public sealed class G<T> { [Key(0)] public T? Value; }
[Union(0, typeof(GuArm))] public interface IGu { }
[MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List<FbArm> Items = new(); }
[MessagePackObject] public sealed class GHold { [Key(0)] public List<FbArm> Items = new(); }
[MessagePackObject] public struct GStruct { [Key(0)] public List<FbArm> Items; }
#pragma warning disable CS0169, CS0414, CS0649
[MessagePackObject] public sealed class DkEl { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; }
[MessagePackObject] public sealed class DkEl2 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; }
#pragma warning restore CS0169, CS0414, CS0649
[MessagePackObject] public sealed class DkEl3 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] public int M; }
[CultDocument("tests.dk_doc3", "tests.dk_doc3.v1")]
[MessagePackObject]
public sealed class DkDoc3 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List<DkEl3> Items = new(); }
[CultDocument("tests.dk_doc", "tests.dk_doc.v1")]
[MessagePackObject]
public sealed class DkDoc { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List<DkEl> Items = new(); }
[CultDocument("tests.dk_doc2", "tests.dk_doc2.v1")]
[MessagePackObject]
public sealed class DkDoc2 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List<DkEl2> Items = new(); }
[MessagePackFormatter(typeof(FormattedRolesFormatter))] public sealed class FormattedRoles : List<FbArm> { }
public sealed class FormattedRolesFormatter : IMessagePackFormatter<FormattedRoles>
{
public void Serialize(ref MessagePackWriter writer, FormattedRoles value, MessagePackSerializerOptions options) => writer.WriteNil();
public FormattedRoles Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new FormattedRoles(); }
}
[DataContract] public sealed class DcSeq : IEnumerable<FbArm>
{
[DataMember(Order = 0)] public List<FbArm> Items = new();
public IEnumerator<FbArm> GetEnumerator() => Items.GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
public interface IArms : IEnumerable<FbArm> { }
public abstract class AbsArms : IEnumerable<FbArm>
{
public abstract IEnumerator<FbArm> GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
}
[MessagePackObject] public sealed class KLL : List<List<FbArm>> { [Key(0)] public int N; }
#pragma warning disable CS0169, CS0414, CS0649
[MessagePackObject(AllowPrivate = true)] public sealed class ApTrueHold { [Key(0)] private List<FbArm> items = new(); }
#pragma warning restore CS0169, CS0414, CS0649
[DataContract] public sealed class DcColl : List<FbArm> { [DataMember(Order = 0)] public int N; }
[DataContract] public sealed class DcBad : List<FbArm> { [DataMember(Order = 0)] public int A; [DataMember(Order = 0)] public int B; }
public sealed class DkFormatter : IMessagePackFormatter<DkEl>
{
public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N);
public DkEl Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => new DkEl { N = reader.ReadInt32() };
}
[MessagePackObject]
public sealed class ApArm
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public int N;
[Key(2)] private int secret;
[IgnoreMember] public int Secret => secret;
public ApArm WithSecret(int value)
{
secret = value;
return this;
}
}
[CultDocument("tests.ap_doc", "tests.ap_doc.v1")]
[MessagePackObject]
public sealed class ApDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<ApArm> Items = new();
}
[MessagePackObject]
public sealed class ApPlainArm
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public int N;
}
[CultDocument("tests.ap_plain_doc", "tests.ap_plain_doc.v1")]
[MessagePackObject]
public sealed class ApPlainDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<ApPlainArm> Items = new();
}
[MessagePackObject]
public sealed class LcArm
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public int N;
}
[MessagePackObject]
public sealed class LcOther
{
[Key(0)] [CultElementId] public string Id = string.Empty;
}
public sealed class LcValue
{
public int N;
}
[CultDocument("tests.lc_doc", "tests.lc_doc.v1")]
[MessagePackObject]
public sealed class LcDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<LcArm> Items = new();
}
[CultDocument("tests.lc_value_doc", "tests.lc_value_doc.v1")]
[MessagePackObject]
public sealed class LcValueDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<LcOther> Items = new();
[Key(2)] public List<LcValue> Values = new();
[Key(3)] public List<Pair> Points = new();
}
[MessagePackObject]
public sealed class FbArm
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public int N;
}
[CultDocument("tests.fb_doc", "tests.fb_doc.v1")]
[MessagePackObject]
public sealed class FbDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<FbArm> Items = new();
}
// Claims List<LcArm> (an object list's type: refused), List<LcValue> and List<Pair> (value lists: accepted, as a math
// resolver claims List<float2>), and answers FbArm with MessagePack's own formatter (no claim).
public sealed class ListClaimResolver : IFormatterResolver
{
public static readonly IFormatterResolver Instance = new ListClaimResolver();
public IMessagePackFormatter<T>? GetFormatter<T>()
{
if (typeof(T) == typeof(List<LcArm>)) return (IMessagePackFormatter<T>)(object)new NilFormatter();
if (typeof(T) == typeof(List<LcValue>)) return (IMessagePackFormatter<T>)(object)new ValuesFormatter();
if (typeof(T) == typeof(List<Pair>)) return (IMessagePackFormatter<T>)(object)new PointsFormatter();
if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter<T>();
if (typeof(T) == typeof(DkEl)) return (IMessagePackFormatter<T>)(object)new DkFormatter();
if (typeof(T) == typeof(DkEl2)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter<T>();
if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List<ApArm>) ||
typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List<ApPlainArm>))
return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter<T>();
return null;
}
private sealed class NilFormatter : IMessagePackFormatter<List<LcArm>>
{
public void Serialize(ref MessagePackWriter writer, List<LcArm> value, MessagePackSerializerOptions options) => writer.WriteNil();
public List<LcArm> Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
reader.Skip();
return new List<LcArm>();
}
}
private sealed class ValuesFormatter : IMessagePackFormatter<List<LcValue>>
{
public void Serialize(ref MessagePackWriter writer, List<LcValue> value, MessagePackSerializerOptions options)
{
writer.WriteArrayHeader(value.Count);
foreach (var item in value) writer.Write(item.N);
}
public List<LcValue> Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
var count = reader.ReadArrayHeader();
var list = new List<LcValue>(count);
for (var index = 0; index < count; index++) list.Add(new LcValue { N = reader.ReadInt32() });
return list;
}
}
private sealed class PointsFormatter : IMessagePackFormatter<List<Pair>>
{
public void Serialize(ref MessagePackWriter writer, List<Pair> value, MessagePackSerializerOptions options)
{
writer.WriteArrayHeader(value.Count);
foreach (var item in value) { writer.WriteArrayHeader(2); writer.Write(item.A); writer.Write(item.B); }
}
public List<Pair> Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
var count = reader.ReadArrayHeader();
var list = new List<Pair>(count);
for (var index = 0; index < count; index++)
{
reader.ReadArrayHeader();
list.Add(new Pair { A = reader.ReadSingle(), B = reader.ReadSingle() });
}
return list;
}
}
}
// Composed after ListClaimResolver: once that resolver has answered FbArm with the standard formatter the composite never
// asks this one, so its claim on the same type must not refuse the document.
public sealed class FbLaterClaimResolver : IFormatterResolver
{
public static readonly IFormatterResolver Instance = new FbLaterClaimResolver();
public IMessagePackFormatter<T>? GetFormatter<T>() => typeof(T) == typeof(FbArm) ? (IMessagePackFormatter<T>)(object)new LaterFormatter() : null;
private sealed class LaterFormatter : IMessagePackFormatter<FbArm>
{
public void Serialize(ref MessagePackWriter writer, FbArm value, MessagePackSerializerOptions options) => writer.WriteNil();
public FbArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
reader.Skip();
return new FbArm();
}
}
}
[Test]
public void OptionsRefuseANullDocumentTypeOrAssembly()
{
Assert.Throws<ArgumentNullException>(() => CultDocumentMessagePackSerialization.OptionsFor(null!));
Assert.Throws<ArgumentNullException>(() => CultDocumentMessagePackSerialization.OptionsForDocument(null!));
}
[Union(0, typeof(ClaimArm))]
public abstract class ClaimNode { }
[MessagePackObject]
public sealed class ClaimArm : ClaimNode
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public int N;
}
[CultDocument("tests.claim_doc", "tests.claim_doc.v1")]
[MessagePackObject]
public sealed class ClaimDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public List<ClaimNode> Items = new();
}
public sealed class ClaimArmResolver : IFormatterResolver
{
public static readonly IFormatterResolver Instance = new ClaimArmResolver();
public IMessagePackFormatter<T>? GetFormatter<T>() => typeof(T) == typeof(ClaimArm) ? (IMessagePackFormatter<T>)(object)new ClaimFormatter() : null;
private sealed class ClaimFormatter : IMessagePackFormatter<ClaimArm>
{
public void Serialize(ref MessagePackWriter writer, ClaimArm value, MessagePackSerializerOptions options)
{
writer.WriteArrayHeader(1);
writer.Write(value.N);
}
public ClaimArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
reader.ReadArrayHeader();
return new ClaimArm { N = reader.ReadInt32() };
}
}
}
// The guard wraps the composite, so the composite's own order is still the contract: a consumer resolver outranks the
// document resolver (CultRecordRef) and MessagePack's own object formatter, here for a type it also knows.
[Test]
@ -568,7 +1097,7 @@ namespace GameCult.Caching.Tests
}
// An arm is [key, armSlots]; the keys here start at 3, skip 0-2, 4, 6 and 7, and never reuse a retired key.
[Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(StubArm))]
[Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(Nine))]
public abstract class GuardNode { }
[MessagePackObject]
@ -592,11 +1121,25 @@ namespace GameCult.Caching.Tests
[MessagePackObject] public sealed class Unit8 : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; }
[MessagePackObject] public sealed class Nine : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; }
// A subtype the union does not name.
[MessagePackObject] public sealed class Rogue : GuardNode { [Key(0)] public string Note = string.Empty; }
// Written and read by a consumer resolver, not by MessagePack.
public sealed class StubArm : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; }
// Written and read by a consumer resolver, not by MessagePack. It can only sit outside a list: a list element is written
// by its [Key] layout, or its id never reaches disk (ListElementsClaimedByAResolverAreRefused).
[Union(9, typeof(StubArm))]
public abstract class StubNode { }
public sealed class StubArm : StubNode { [Key(0)] [CultElementId] public string Id = string.Empty; }
[CultDocument("tests.stub_doc", "tests.stub_doc.v1")]
[MessagePackObject]
public sealed class StubDoc
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public StubNode? Root;
}
public sealed class StubArmResolver : IFormatterResolver
{

14
tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs
View file

@ -5,6 +5,7 @@ using GameCult.Caching.MessagePack;
using GameCult.Mesh;
using GameCult.Networking;
using MessagePack;
using MessagePack.Resolvers;
using NUnit.Framework;
using R3;
using System;
@ -193,6 +194,19 @@ namespace GameCult.Geometry.Tests
.Should().Be("geometry:domain:02c9b5810977406b0c206f3a3494327a423abb9448192be1b9a1863cd0f2ed95");
}
// A source-generated formatter is of no known family and passes the element-id door (ruling c2a-source-generated-formatters-defaulted).
[Test]
public void AGeneratedFormatterPassesTheElementIdDoor()
{
StandardResolver.Instance.GetFormatter<CultGeometryDomainNode>().Should()
.NotBeSameAs(DynamicObjectResolver.Instance.GetFormatter<CultGeometryDomainNode>(), "the fixture must stay source-generated");
var payload = File.ReadAllBytes(FixturePath("ragnarok-domain.msgpack"));
var domain = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultGeometryDomainDocument), payload);
var again = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(
typeof(CultGeometryDomainDocument), CultDocumentMessagePackSerialization.SerializeUntyped(domain, typeof(CultGeometryDomainDocument)));
CultGeometryDomainDocument.CreateRecordKey(again).Value.Should().Be(CultGeometryDomainDocument.CreateRecordKey(domain).Value);
}
[Test]
public void RustExportedChunkArtifact_DecodesWithStableCultCacheKey()
{

126
tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using FluentAssertions;
@ -171,4 +172,129 @@ public sealed class CultMeshSingleFileElementIdTests
});
cache.Get<CultMeshBodyPublicationDocument>(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first);
}
// A list document defined in this assembly: the record another writer leaves holding ids under an unmarked header.
[CultDocument("mesh.test.id_list", "mesh.test.id_list.v1")]
[MessagePackObject]
public sealed class MeshIdList
{
[Key(0)] [CultName] public string Name { get; set; } = "";
[Key(1)] public List<MeshIdItem> Items { get; set; } = new();
}
[MessagePackObject]
public sealed class MeshIdItem
{
[Key(0)] public string N { get; set; } = "";
[Key(1)] [CultElementId] public string Id { get; set; } = "";
}
private static readonly CultRecordKey IdsKey = new("ids");
private const string HexB = "bbbbbbbbbbbb";
// A v1 file whose record holds 12-hex element ids: the document is written with the type (v3), then the header is put back
// to the v1 a writer that does not decide would have left.
private string UnmarkedStoreHoldingIds()
{
var path = Path_("unmarked.cc");
CultMesh.WriteSingleFileDocument(
path,
IdsKey,
new MeshIdList { Name = "ids", Items = { new MeshIdItem { Id = HexA }, new MeshIdItem { Id = HexB } } },
registry: Registry(typeof(MeshIdList)));
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
HeaderOf(path).Should().Be("cultcache.store.v1");
return path;
}
private string PythonStore()
{
string? vectors = null;
for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null && vectors == null; dir = dir.Parent)
{
var candidate = Path.Combine(dir.FullName, "tests", "vectors", "numeric-leaves", "py-store", "leaf-record.cc");
if (File.Exists(candidate)) vectors = candidate;
}
vectors.Should().NotBeNull("the shared vectors sit above the test directory");
var path = Path_("leaf-record.cc");
File.Copy(vectors!, path);
HeaderOf(path).Should().Be("cultcache.store.v1", "cultcache-py leaves the header unmarked");
return path;
}
private static CultDocumentRegistry Registry(params Type[] types) => CultDocumentRegistry.ForTypes(types);
// A CultMesh write is a writer with a type: it decides for the copied record from the bytes the file holds, whatever any
// earlier read saw (a fresh store per call has read nothing).
[Test]
public void ACultMeshWriteThatCopiesARecordHoldingIdsMarksTheStore()
{
var path = UnmarkedStoreHoldingIds();
CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument)));
HeaderOf(path).Should().Be("cultcache.store.v3");
CultMesh.ReadSingleFileDocument<MeshIdList>(path, IdsKey, Registry(typeof(MeshIdList))).Items.Select(item => item.Id).Should().Equal(HexA, HexB);
}
// A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read.
[Test]
public void ARawCultMeshWriteBesideARecordHoldingIdsLeavesTheHeaderAlone()
{
var path = UnmarkedStoreHoldingIds();
CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 });
HeaderOf(path).Should().Be("cultcache.store.v1");
}
// Soul s1 P8, committed: the writer's registry has only the type it writes, so the python record is foreign to it and it
// decides nothing for it (variants:question:non-csharp-writers-mark-ids-2).
[Test]
public void ACultMeshWriteWhoseRegistryLacksTheRecordsTypeDecidesNothingForIt()
{
var path = PythonStore();
CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument)));
HeaderOf(path).Should().Be("cultcache.store.v1");
}
// The copy check never refuses a write: a record the registry claims by schema name but cannot resolve (the catalog entry
// names another type for a slot) is copied as its bytes and decides nothing. Soul s2 P5, incompatible-catalog-slot12.
[Test]
public void AnUnresolvableCopiedRecordIsCopiedAndDecidesNothing()
{
var path = UnmarkedStoreHoldingIds();
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
const string changedId = "mesh.test.id_list.v0";
var entry = snapshot.SchemaCatalog.Single(candidate => candidate.SchemaName == "mesh.test.id_list");
entry.SchemaId = changedId;
entry.ContentHash = "persisted-by-another-runtime";
entry.Members.Single(member => member.Slot == 1).TypeName = "System.Collections.Generic.List`1[[Other.Inner]]";
snapshot.Records.Single(record => record.Key == IdsKey.Value).SchemaId = changedId;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value);
Assert.DoesNotThrow(() => CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument))));
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value);
after.SchemaId.Should().Be(changedId);
after.Payload.Should().Equal(before.Payload);
HeaderOf(path).Should().Be("cultcache.store.v1");
}
// Typed writes beside records that hold no id leave the store unmarked: the check marks only on an id it found.
[Test]
public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked()
{
var path = Path_("plain.cc");
CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument)));
CultMesh.WriteSingleFileDocument(path, new CultRecordKey("second"), Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument)));
HeaderOf(path).Should().Be("cultcache.store.v1");
}
}