CultLib/src/GameCult.Caching/CultElementIds.cs
Meta Vens 73b4b47451 CultElementIds: collection types join the id path, member attributes read through overrides, each unkeyed element type refused once
Graph edges and lists carry the collection types between member and child; Settle adds them to the path so
[MessagePackFormatter] on a collection class and a resolver claim on List<Elem> are judged. Declares replaces the four
MemberInfo.IsDefined member reads, which ignore inherit for properties. Survey.Judged dedupes the unkeyed-element refusal.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-15 00:26:52 +02:00

563 lines
27 KiB
C#

using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using MessagePack;
namespace GameCult.Caching
{
// The identity of one element of an object list. A string member, keyed like any other, on every object type that
// sits in a list of a registered document type (union bases included: every concrete subtype carries the member).
// [CultElementId(nameof(Offset))] declares the id derived, never random: an unset id becomes the invariant-culture text of
// that member (a keyed string or number on the same type, unique within its list). Content-addressed elements use it so the
// same content always encodes to the same bytes.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)]
public sealed class CultElementIdAttribute : Attribute
{
public CultElementIdAttribute(string? derivedFrom = null)
{
DerivedFrom = derivedFrom;
}
public string? DerivedFrom { get; }
}
// A record the id rule refuses: a duplicate id in one list, a random id that is not 12 lowercase hex characters, or a derived
// id whose source member is null or empty. It names the record, the list (record key, member slot, dictionary key and
// element index) and, where there is one, the element id and the source member.
public sealed class CultElementIdException : InvalidOperationException
{
public CultElementIdException(string message, string recordKey, string listPath, string? elementId = null, string? member = null)
: base(message)
{
RecordKey = recordKey;
ListPath = listPath;
ElementId = elementId;
Member = member;
}
public string RecordKey { get; }
public string ListPath { get; }
public string? ElementId { get; }
public string? Member { get; }
}
// The one owner of element identity: which types need an id, and how unset ids are filled.
// A random id is 12 lowercase hex characters (48 bits); a derived id is the invariant text of its source and only has to be
// non-empty. Either need only be unique within one list. A write mints a random id; a load of a store written before ids
// existed mints from SHA-256(record key, element path), so every reader of the same store mints the same id and a reload is
// identical to what was read. A write and a load refuse the same things.
internal static class CultElementIds
{
private const int IdBytes = 6;
// The ids a pass would fill, decided before anything is written: nothing changes until Apply, and Undo restores what
// Apply replaced.
internal sealed class IdPlan
{
private readonly List<(object Item, Shape Shape, string? Before, string Id)> _fills = new();
public int Count => _fills.Count;
internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id));
public void Apply()
{
foreach (var (item, shape, _, id) in _fills)
shape.SetId!(item, id);
}
public void Undo()
{
foreach (var (item, shape, before, _) in _fills)
shape.SetId!(item, before!);
}
}
internal sealed class Shape
{
public Func<object, string?>? GetId;
public Action<object, string>? SetId;
public Func<object, string?>? Derive;
public string? DerivedFrom;
public (int Slot, Func<object, object?> Get)[] Walk = Array.Empty<(int, Func<object, object?>)>();
}
private static readonly ConcurrentDictionary<Type, Shape> Shapes = new();
// ---- registration: which element types must carry an id ----
internal sealed class Survey
{
public List<string> Problems { get; } = new();
// Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's
// element types and every concrete type under them. A serialization codec must write all of these by their [Key]
// layout, so it checks them against the formatters it composed.
public List<Type> IdPath { get; } = new();
// The concrete element types already judged in this document, so each is refused at most once however many lists hold it.
internal HashSet<Type> Judged { get; } = new();
}
// What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type
// can sit in a cycle, so the path is a fixed point over these edges and not a stack.
private sealed class Graph
{
public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new();
public List<(Type Owner, MemberInfo Member, Type Element, List<Type> Concrete, Type[] Through)> Lists { get; } = new();
}
private static readonly ConcurrentDictionary<Type, bool> IdCapable = new();
// A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask.
internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0);
// Every reason this document type cannot be registered under the id rule, one per offending concrete element type.
internal static Survey Inspect(Type documentType)
{
var survey = new Survey();
var graph = new Graph();
VisitObject(documentType, documentType, new HashSet<Type>(), survey, graph);
Settle(documentType, survey, graph);
return survey;
}
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, Survey survey, Graph graph)
{
if (!seen.Add(type))
return;
foreach (var (member, memberType) in KeyedMembers(type))
VisitMemberType(documentType, type, member, memberType, Array.Empty<Type>(), seen, survey, graph);
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
{
graph.Edges.Add((type, null, union.SubType, Array.Empty<Type>()));
VisitObject(documentType, union.SubType, seen, survey, graph);
}
}
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet<Type> seen, Survey survey, Graph graph)
{
memberType = Nullable.GetUnderlyingType(memberType) ?? memberType;
if (IsLeaf(memberType))
return;
if (DictionaryValueType(memberType) is { } valueType)
{
VisitMemberType(documentType, owner, member, valueType, through.Append(memberType).ToArray(), seen, survey, graph);
return;
}
if (ElementTypeOf(memberType) is { } element)
{
if (IsObjectType(element))
{
var concretes = ConcreteTypes(element).ToList();
graph.Lists.Add((owner, member, element, concretes, through.Append(memberType).ToArray()));
foreach (var concrete in concretes)
{
if (!survey.Judged.Add(concrete))
continue;
var problem = IsObjectType(concrete) ? IdProblem(concrete) : UnkeyedElementProblem;
if (problem != null)
survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}");
}
VisitObject(documentType, element, seen, survey, graph);
}
else
VisitMemberType(documentType, owner, member, element, through.Append(memberType).ToArray(), seen, survey, graph);
return;
}
if (IsObjectType(memberType))
{
graph.Edges.Add((owner, member, memberType, through));
VisitObject(documentType, memberType, seen, survey, graph);
}
}
private const string UnkeyedElementProblem =
"is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists.";
// The id path: every type that reaches an object list, the list's element types and the concrete types under them.
// [MessagePackFormatter] on any of them, or on a member the walk descends through, would hand the bytes to a formatter
// that does not write the [Key] layout the id lives in.
private static void Settle(Type documentType, Survey survey, Graph graph)
{
var reaching = new HashSet<Type>(graph.Lists.Select(list => list.Owner));
var members = new List<MemberInfo>(graph.Lists.Select(list => list.Member));
var through = new List<Type>(graph.Lists.SelectMany(list => list.Through));
for (var grew = true; grew;)
{
grew = false;
foreach (var (owner, member, child, via) in graph.Edges)
{
if (!reaching.Contains(child))
continue;
if (member != null && !members.Contains(member))
members.Add(member);
through.AddRange(via);
grew |= reaching.Add(owner);
}
}
var path = new List<Type>();
void Add(Type type)
{
if (!path.Contains(type))
path.Add(type);
}
foreach (var type in reaching)
Add(type);
foreach (var type in through)
Add(type);
foreach (var list in graph.Lists)
{
Add(list.Element);
foreach (var concrete in list.Concrete)
Add(concrete);
}
foreach (var type in path)
{
survey.IdPath.Add(type);
if (type.IsDefined(typeof(MessagePackFormatterAttribute), true))
survey.Problems.Add($"Cult document {documentType.Name}: {type.Name} is on the path to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute or keep this type out of the document's lists.");
}
foreach (var member in members)
{
if (Declares(member, typeof(MessagePackFormatterAttribute)))
survey.Problems.Add($"Cult document {documentType.Name}: {member.DeclaringType!.Name}.{member.Name} leads to a list whose elements carry ids and declares [MessagePackFormatter], so its element ids would never reach disk; remove the attribute.");
}
}
// The types an element can actually be: the element type itself when it can be instantiated, and every union subtype
// under it. Shapes are built from these, so registration judges exactly what minting will meet.
private static IEnumerable<Type> ConcreteTypes(Type element)
{
var seen = new HashSet<Type>();
return Expand(element).ToList();
IEnumerable<Type> Expand(Type type)
{
if (!seen.Add(type))
yield break;
if (!type.IsAbstract && !type.IsInterface)
yield return type;
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
{
foreach (var subtype in Expand(union.SubType))
yield return subtype;
}
}
}
private static string? IdProblem(Type element)
{
var ids = MembersOf(element).Where(member => Declares(member, typeof(CultElementIdAttribute))).ToArray();
if (ids.Length == 0)
return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity.";
if (ids.Length > 1)
return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity.";
var id = ids[0];
if (Declares(id, typeof(MessagePackFormatterAttribute)))
return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key].";
if (TypeOf(id) != typeof(string))
return $"declares [CultElementId] on {id.Name}, which is not a string.";
if (KeyOf(id) is not { } idKey)
return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted.";
var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey);
if (sharing.Member != null)
return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own.";
if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true })
return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it.";
var source = id.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom;
if (source != null)
{
var from = MembersOf(element).FirstOrDefault(member => member.Name == source);
if (from == null || KeyOf(from) == null)
return $"derives its [CultElementId] from {source}, which is not a keyed member of the type.";
var kind = Nullable.GetUnderlyingType(TypeOf(from)) ?? TypeOf(from);
if (kind != typeof(string) && !kind.IsPrimitive)
return $"derives its [CultElementId] from {source}, which is not a string or a number.";
}
return null;
}
// ---- write: what the stored form holds ----
// True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value
// as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write.
internal static bool Holds(object? root)
{
if (root == null)
return false;
var type = root.GetType();
if (IsLeaf(type))
return false;
if (root is IDictionary dictionary)
{
foreach (DictionaryEntry entry in dictionary)
{
if (Holds(entry.Value))
return true;
}
return false;
}
if (root is IEnumerable list)
{
foreach (var item in list)
{
if (item != null && IsObjectType(item.GetType()) && ShapeOf(item.GetType()).GetId is { } getId &&
!string.IsNullOrEmpty(getId(item)))
return true;
if (Holds(item))
return true;
}
return false;
}
return IsObjectType(type) && ShapeOf(type).Walk.Any(member => Holds(member.Get(root)));
}
// ---- write and load: mint and check ----
// Decides every unset element id under root, and refuses what the rule refuses, without changing anything: a random id
// that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty.
// rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits
// twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load.
internal static IdPlan Plan(object? root, string rootPath, bool deterministic)
{
var plan = new IdPlan();
Visit(root, rootPath);
return plan;
void Visit(object? value, string path)
{
if (value == null)
return;
var type = value.GetType();
if (IsLeaf(type))
return;
if (value is IDictionary dictionary)
{
foreach (DictionaryEntry entry in dictionary)
Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}");
}
else if (value is IEnumerable list)
Elements(list, path);
else if (IsObjectType(type))
{
foreach (var (slot, get) in ShapeOf(type).Walk)
Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture));
}
}
void Elements(IEnumerable list, string path)
{
var items = list.Cast<object?>().ToArray();
var taken = new HashSet<string>(StringComparer.Ordinal);
var objects = new HashSet<object>(ReferenceComparer.Instance);
foreach (var item in items)
{
if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape)
continue;
if (!objects.Add(item))
throw new CultElementIdException(
$"Record {rootPath}: the same {item.GetType().Name} object appears twice in the list at {path}; an element is one object with one id.",
rootPath, path);
var id = getId(item);
if (string.IsNullOrEmpty(id))
continue;
if (shape.Derive == null && !IsRandomId(id!))
throw new CultElementIdException(
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].",
rootPath, path, id);
if (!taken.Add(id!))
throw Duplicate(id!, path);
}
for (var index = 0; index < items.Length; index++)
{
var item = items[index];
if (item == null)
continue;
var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture);
if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape &&
string.IsNullOrEmpty(get(item)))
{
string id;
if (itemShape.Derive != null)
{
id = itemShape.Derive(item) ?? string.Empty;
if (id.Length == 0)
throw new CultElementIdException(
$"Record {rootPath}: the element at {itemPath} derives its id from {itemShape.DerivedFrom}, which is null or empty.",
rootPath, itemPath, member: itemShape.DerivedFrom);
if (!taken.Add(id))
throw Duplicate(id, path);
}
else
id = Mint(itemPath, deterministic, rootPath, taken);
plan.Add(item, itemShape, get(item), id);
}
Visit(item, itemPath);
}
}
CultElementIdException Duplicate(string id, string path) => new(
$"Record {rootPath}: element id '{id}' appears twice in the list at {path}; an element id is unique within its list.",
rootPath, path, id);
}
private sealed class ReferenceComparer : IEqualityComparer<object>
{
public static readonly ReferenceComparer Instance = new();
bool IEqualityComparer<object>.Equals(object? left, object? right) => ReferenceEquals(left, right);
int IEqualityComparer<object>.GetHashCode(object value) => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(value);
}
private static bool IsRandomId(string id)
{
if (id.Length != IdBytes * 2)
return false;
foreach (var character in id)
{
if (!(character is >= '0' and <= '9' or >= 'a' and <= 'f'))
return false;
}
return true;
}
private static string Mint(string itemPath, bool deterministic, string rootPath, HashSet<string> taken)
{
for (var attempt = 0; ; attempt++)
{
var bytes = new byte[IdBytes];
if (deterministic)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt.ToString(CultureInfo.InvariantCulture)));
Array.Copy(hash, bytes, IdBytes);
}
else
RandomBytes(bytes);
var id = Hex(bytes);
if (taken.Add(id))
return id;
}
}
private static void RandomBytes(byte[] bytes)
{
using var random = RandomNumberGenerator.Create();
random.GetBytes(bytes);
}
private static string Hex(byte[] bytes)
{
var text = new StringBuilder(bytes.Length * 2);
foreach (var value in bytes)
text.Append(value.ToString("x2", CultureInfo.InvariantCulture));
return text.ToString();
}
// ---- shapes ----
private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build);
private static Shape Build(Type type)
{
var shape = new Shape();
var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute)));
if (idMember != null)
{
shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o);
shape.SetId = idMember is FieldInfo f ? (o, v) => f.SetValue(o, v) : (o, v) => ((PropertyInfo)idMember).SetValue(o, v);
if (idMember.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom is { } source &&
MembersOf(type).FirstOrDefault(member => member.Name == source) is { } from)
{
shape.DerivedFrom = source;
shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), CultureInfo.InvariantCulture);
}
}
shape.Walk = KeyedMembers(type)
.Where(entry => !IsLeaf(Nullable.GetUnderlyingType(entry.Type) ?? entry.Type))
.Select(entry => (Slot: KeyOf(entry.Member)!.Value, Get: entry.Member is FieldInfo fi
? (Func<object, object?>)(o => fi.GetValue(o))
: o => ((PropertyInfo)entry.Member).GetValue(o)))
.ToArray();
return shape;
}
private static IEnumerable<MemberInfo> MembersOf(Type type) =>
type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast<MemberInfo>()
.Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod != null));
private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) =>
MembersOf(type).Where(member => KeyOf(member) != null).Select(member => (member, TypeOf(member)));
// An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an
// override of a base virtual member would not see its base's attribute.
private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true);
private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute<KeyAttribute>(true)?.IntKey;
private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType;
// ---- type classification ----
// A value MessagePack writes whole and that cannot hold an element list. Structs are values: never elements, never walked.
private static bool IsLeaf(Type type)
{
if (type.IsPrimitive || type.IsEnum || type == typeof(string) || type.IsValueType)
return true;
return type.IsArray && IsLeaf(type.GetElementType()!);
}
// An object a store can hold an element list under, or hold as an element: a class MessagePack serializes by key.
private static bool IsObjectType(Type type) =>
(type.IsClass || type.IsInterface) && type != typeof(string) && type != typeof(object) &&
(type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true));
private static Type? ElementTypeOf(Type type)
{
if (type.IsArray)
return type.GetElementType();
foreach (var candidate in type.GetInterfaces().Append(type))
{
if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IEnumerable<>))
return candidate.GetGenericArguments()[0];
}
return null;
}
private static Type? DictionaryValueType(Type type)
{
foreach (var candidate in type.GetInterfaces().Append(type))
{
if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>))
return candidate.GetGenericArguments()[1];
}
return null;
}
}
}