GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

CultCache: element id rules, atomic minting, minted override values, the v3 marker

Browse source 
The id rule is one plan, made before anything is written: Plan decides every fill and
refuses a duplicate, a random id that is not 12 lowercase hex characters, and a derived
id whose source is null or empty, naming the record and list (CultElementIdException).
A load refuses what a write refuses. Apply and Undo make minting part of the admission:
a refused or uncommitted write leaves the caller's documents as they were.

Override values mint inside Resolve, into the decoded copy, so the public
CultVariantOverride.Set, C1-era overrides and Override() all get ids on write and on load;
Override() no longer mints. Interface unions are judged on their concrete types, the same
rule registration and minting share, and a subtype reusing the id's key is refused at
registration. Dictionary keys format with the invariant culture.

Records minted at load are marked per entry (IdsInMemoryOnly) instead of listed by key, so
a deleted record leaves the mark with it, and a flush of the whole store clears it. A store
whose types reach an object list declares cultcache.store.v3 (directory: v5), which every
reader older than ids refuses; a marked file stays marked when a writer merges onto it.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-29 18:27:16 +02:00
commit fc52f984db
4 changed files with 294 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
View file

@ -185,7 +185,7 @@ public static class CultDocumentMessagePackSerialization
if (fieldCount > 0)
{
snapshot.FormatVersion = reader.ReadString()
?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + ".");
?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + ".");
}
if (fieldCount > 1)
@ -221,20 +221,21 @@ public static class CultDocumentMessagePackSerialization
{
var version = snapshot.FormatVersion;
if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) &&
!string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal))
!string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal) &&
!string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal))
{
throw new NotSupportedException(
$"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}.");
$"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}.");
}
// Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust.
// Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust.
if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal))
{
var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null);
if (variant != null)
{
throw new NotSupportedException(
$"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2}.");
$"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2} or {CultPersistedStoreSnapshot.FormatV3}.");
}
}
}

17
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -16,6 +16,9 @@ namespace GameCult.Caching.MessagePack;
public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
{
private const string IndexedFormatVersion = "cultcache.store.v4.directory-content-addressed-pages";
// The manifest of a directory that holds element ids: a reader older than ids refuses it, because it would skip the id
// slots of a page and rewrite the elements without them.
private const string IndexedFormatVersionWithIds = "cultcache.store.v5.directory-content-addressed-pages";
// An unleased load that has not settled after this many attempts throws.
private const int UnleasedLoadAttempts = 5;
private readonly FileInfo _manifestFile;
@ -283,9 +286,12 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
pagePayload);
}
// A manifest already marked stays marked: pages it names may hold ids this cache cannot read.
var carriesIds = string.Equals(currentManifest.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal) ||
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.Descriptor.CarriesElementIds);
WriteManifest(targetCatalog, currentIndex.Values
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray());
.ToArray(), carriesIds);
DeleteUnreferencedRecordPages(currentIndex.Values);
@ -295,11 +301,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
MarkFlushSucceeded();
}
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index)
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, bool carriesElementIds)
{
var manifest = new CultPersistedStoreSnapshot
{
FormatVersion = IndexedFormatVersion,
FormatVersion = carriesElementIds ? IndexedFormatVersionWithIds : IndexedFormatVersion,
SchemaCatalog = catalog,
Records = index
};
@ -444,10 +450,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes);
if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal))
if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) &&
!string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal))
{
throw new InvalidOperationException(
$"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} is readable.");
$"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.");
}
return snapshot;

156
src/GameCult.Caching/CultCache.cs
View file

@ -64,6 +64,7 @@ namespace GameCult.Caching
{
public const string FormatV1 = "cultcache.store.v1";
public const string FormatV2 = "cultcache.store.v2";
public const string FormatV3 = "cultcache.store.v3";
public string FormatVersion { get; set; } = FormatV1;
public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty<CultSchemaCatalogEntry>();
@ -210,6 +211,9 @@ namespace GameCult.Caching
internal IReadOnlyList<CultDocumentMemberDescriptor> Members { get; }
internal IReadOnlyList<CultDocumentRegistry.PersistedMember> RichMembers { get; }
// True when the type reaches an object list: what it writes can hold element ids, so a store holding it carries the id marker.
public bool CarriesElementIds { get; internal set; }
// The public read surface (CultNet selection cut 1, section 6): declared addressing, and the
// three ways to read a value through it. GameCult.Networking has no InternalsVisibleTo here on
// purpose - this is the only door, so nothing outside the cache decides what a member means.
@ -376,9 +380,11 @@ namespace GameCult.Caching
// A plain record staged by Flatten: the one way a plain record may replace a variant at its key.
internal bool Flattens { get; set; }
internal bool IsResolved => _document != null;
// True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds).
internal bool IdsInMemoryOnly { get; set; }
internal CultStoredDocument Resolved(object document) => new(Key, StoredAt, Descriptor, Variant!, document);
internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly) =>
new(Key, StoredAt, Descriptor, delta, document) { IdsInMemoryOnly = idsInMemoryOnly };
}
public sealed class CultDocumentRegistry
@ -632,8 +638,9 @@ namespace GameCult.Caching
?? throw new InvalidOperationException(
$"Type {type.FullName} is not marked with {nameof(CultDocumentAttribute)}.");
var members = DiscoverMembers(type);
if (CultElementIds.Problems(type) is { Count: > 0 } idProblems)
throw new InvalidOperationException(idProblems[0]);
var ids = CultElementIds.Inspect(type);
if (ids.Problems.Count > 0)
throw new InvalidOperationException(ids.Problems[0]);
var nameMember = members.FirstOrDefault(member => member.IsName);
var indexAccessors = members
.Where(member => member.IndexAlias != null)
@ -671,7 +678,10 @@ namespace GameCult.Caching
nameMember?.GetterNullable,
indexAccessors,
descriptorMembers,
members);
members)
{
CarriesElementIds = ids.CarriesIds
};
}
private static string BuildSemanticFingerprint(
@ -1800,8 +1810,6 @@ namespace GameCult.Caching
private readonly CultDocumentRegistry _registry;
private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new();
private readonly Dictionary<string, CultStoredDocument> _entries = new(StringComparer.Ordinal);
// Keys whose element ids were minted at load and not yet written (see MintElementIds).
private readonly HashSet<string> _mintedOnLoad = new(StringComparer.Ordinal);
private readonly Dictionary<Type, Dictionary<string, string>> _names = new();
// Every key holding a value (unordered): the index is truthful about duplicates, so removing one holder never hides another.
private readonly Dictionary<(Type Type, string Alias), Dictionary<string, List<string>>> _indexes = new();
@ -1830,6 +1838,19 @@ namespace GameCult.Caching
// Read under the gate, by a store judging a merge.
internal bool HoldsVariants => _variantKeys.Count > 0;
// A store wrote its whole view: a plain record it wrote carries the ids it held, so it is no longer in-memory only. A
// variant is not cleared here: the store wrote the delta it was handed, and only a write of the variant changes that.
internal void IdsPersisted(IEnumerable<string> keys) => Held(() =>
{
foreach (var key in keys)
{
if (_entries.TryGetValue(key, out var entry) && entry.Variant == null)
entry.IdsInMemoryOnly = false;
}
return true;
});
public CultCodec? Codec => _codec;
// The whole override of one member, encoded with this cache's codec, ready for UpsertVariant.
@ -1838,8 +1859,6 @@ namespace GameCult.Caching
if (_codec == null) throw new InvalidOperationException("This cache has no codec; create it with CultCacheMessagePack.Create.");
var found = _registry.GetRequired(documentType).RichMembers.FirstOrDefault(entry => entry.Member.Name == member)
?? throw new ArgumentException($"{documentType.Name} has no persisted member '{member}'.", nameof(member));
if (value != null)
CultElementIds.Assign(value, string.Empty, deterministic: false, refuseDuplicates: true);
return CultVariantOverride.Set(
found.Slot,
_codec.Serialize(value!, found.MemberType, documentType));
@ -1852,18 +1871,22 @@ namespace GameCult.Caching
// it rewrote; a second call finds none.
public int MintElementIds()
{
var groups = Held(() => _mintedOnLoad
.Select(key => _entries.TryGetValue(key, out var entry) ? entry : null)
.Where(entry => entry is { Variant: null })
.GroupBy(entry => Home(entry!.Descriptor.DocumentType))
.Select(group => group.Select(entry => entry!).ToArray())
var groups = Held(() => _entries.Values
.Where(entry => entry.IdsInMemoryOnly)
.GroupBy(entry => Home(entry.Descriptor.DocumentType))
.Select(group => group.ToArray())
.ToArray());
foreach (var group in groups)
{
Commit(batch =>
{
foreach (var entry in group)
batch.Upsert(entry.Descriptor.DocumentType, entry.Document, entry.Key);
{
if (entry.Variant is { } delta)
batch.UpsertVariant(entry.Key, new CultRecordKey(delta.BaseKey), delta.Overrides);
else
batch.Upsert(entry.Descriptor.DocumentType, entry.Document, entry.Key);
}
});
}
@ -2348,30 +2371,47 @@ namespace GameCult.Caching
if (_held == null)
throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold.");
var home = Validate(admitted, evicted, source);
// Ids are filled before anything lands: a write mints random ids and refuses a duplicate in one list; a load of a
// record written before ids existed mints the ids its reload will mint again, and lands in memory only.
var mintedOnLoad = new List<string>();
foreach (var stored in admitted)
// Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that
// refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back.
// A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint
// again, and lands in memory only (variants mint inside Resolve, into their override values).
var idPlans = admitted
.Where(stored => stored.Variant == null)
.Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Document, stored.Key.Value, deterministic: source != null)))
.ToArray();
VariantPlan plan;
CultCommitOutcome outcome;
try
{
if (stored.Variant == null && stored.IsResolved &&
CultElementIds.Assign(stored.Document, stored.Key.Value, deterministic: source != null, refuseDuplicates: source == null) > 0 &&
source != null)
mintedOnLoad.Add(stored.Key.Value);
foreach (var (_, idPlan) in idPlans)
idPlan.Apply();
// The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch
// set, before land and before memory. land receives the admitted records with variants resolved.
plan = Resolve(admitted, evicted, source);
outcome = land(home, plan.Admitted);
}
catch
{
foreach (var (_, idPlan) in idPlans)
idPlan.Undo();
throw;
}
// The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch
// set, before land and before memory. land receives the admitted records with variants resolved.
var plan = Resolve(admitted, evicted, source);
var outcome = land(home, plan.Admitted);
if (outcome != CultCommitOutcome.Committed)
{
foreach (var (_, idPlan) in idPlans)
idPlan.Undo();
return outcome;
// A record whose ids exist only in memory stays listed until a write persists them or the record goes.
foreach (var stored in admitted)
_mintedOnLoad.Remove(stored.Key.Value);
foreach (var stored in evicted)
_mintedOnLoad.Remove(stored.Key.Value);
foreach (var key in mintedOnLoad)
_mintedOnLoad.Add(key);
}
// A record whose ids exist only in memory stays marked until a write persists them (a flush of the whole store
// does, and so does any write of the record: a new entry is unmarked).
if (source != null)
{
foreach (var (stored, idPlan) in idPlans)
stored.IdsInMemoryOnly = idPlan.Count > 0;
}
foreach (var change in Apply(plan, evicted, source))
_held.Add((change, source != null));
return CultCommitOutcome.Committed;
@ -2623,6 +2663,8 @@ namespace GameCult.Caching
var type = descriptor.DocumentType;
var kept = new List<KeyValuePair<int, byte[]>>();
var overrides = new List<CultVariantOverride>(delta.Overrides.Count);
var minted = false;
var seen = new HashSet<int>();
foreach (var entry in delta.Overrides)
{
@ -2638,9 +2680,10 @@ namespace GameCult.Caching
if (member == null)
throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have.");
object? value;
try
{
_codec.Deserialize(member.MemberType, type, entry.Value);
value = _codec.Deserialize(member.MemberType, type, entry.Value);
}
catch (Exception exception)
{
@ -2650,7 +2693,19 @@ namespace GameCult.Caching
exception.GetBaseException().Message, exception);
}
kept.Add(new KeyValuePair<int, byte[]>(slot, entry.Value));
// The override's value is minted like a plain record's members: ids the store never held are filled here, in the
// decoded copy, and the delta this hold lands carries them.
var bytes = entry.Value;
var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null);
if (idPlan.Count > 0)
{
idPlan.Apply();
bytes = _codec.Serialize(value!, member.MemberType, type);
minted = true;
}
overrides.Add(new CultVariantOverride(entry.Op, entry.Path, entry.Id, bytes));
kept.Add(new KeyValuePair<int, byte[]>(slot, bytes));
}
var payload = _codec.Serialize(baseStored.Document, type, type);
@ -2674,7 +2729,7 @@ namespace GameCult.Caching
stack.RemoveAt(stack.Count - 1);
return done[key] = isVariant
? variant.Resolved(document)
? variant.Resolved(document, minted ? new CultVariantDelta(delta.BaseKey, overrides) : delta, variant.IdsInMemoryOnly || (minted && source != null))
: new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { Flattens = true };
}
}
@ -3168,7 +3223,9 @@ namespace GameCult.Caching
{
WriteSnapshot(
Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)),
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()));
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()),
Entries.Values.Any(entry => entry.Descriptor.CarriesElementIds));
Cache?.IdsPersisted(Entries.Keys);
}
MarkFlushSucceeded();
@ -3210,7 +3267,14 @@ namespace GameCult.Caching
foreach (var entry in request.Upserts)
records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload);
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())));
// A file already marked stays marked: it may hold records this cache cannot read, and a rewrite must not shed the marker.
var carriesIds = request.Upserts.Any(entry => entry.Descriptor.CarriesElementIds) ||
(ontoDisk
? string.Equals(disk.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal)
: Entries.Values.Any(entry => entry.Descriptor.CarriesElementIds));
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), carriesIds);
if (!ontoDisk)
Cache?.IdsPersisted(records.Keys);
foreach (var entry in request.Deletes)
Entries.TryRemove(entry.Key.Value, out _);
foreach (var entry in request.Upserts)
@ -3263,16 +3327,20 @@ namespace GameCult.Caching
}
}
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog)
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool carriesElementIds)
{
var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray();
var used = new HashSet<string>(ordered.Select(record => record.SchemaId), StringComparer.Ordinal);
var snapshot = new CultPersistedStoreSnapshot
{
// v2 exactly when the store holds a variant: a store without one stays byte-identical v1.
FormatVersion = ordered.Any(record => record.Variant != null)
? CultPersistedStoreSnapshot.FormatV2
: CultPersistedStoreSnapshot.FormatV1,
// v3 when the store can hold element ids: a reader older than ids refuses the header, because it would skip the id
// slots and rewrite the elements without them. Otherwise v2 exactly when the store holds a variant, and a store
// with neither stays byte-identical v1.
FormatVersion = carriesElementIds
? CultPersistedStoreSnapshot.FormatV3
: ordered.Any(record => record.Variant != null)
? CultPersistedStoreSnapshot.FormatV2
: CultPersistedStoreSnapshot.FormatV1,
SchemaCatalog = catalog
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.First())

213
src/GameCult.Caching/CultElementIds.cs
View file

@ -2,6 +2,7 @@ using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Security.Cryptography;
@ -11,7 +12,7 @@ using MessagePack;
namespace GameCult.Caching
{
// The identity of one element of an object list. A string member, keyed like any other, on every object type that
// sits in a list of a registered document type (union bases included: subclasses inherit the member).
// sits in a list of a registered document type (union bases included: every concrete subtype carries the member).
// [CultElementId(nameof(Offset))] declares the id derived, never random: an unset id becomes the invariant-culture text of
// that member (a keyed string or number on the same type, unique within its list). Content-addressed elements use it so the
// same content always encodes to the same bytes.
@ -26,19 +27,64 @@ namespace GameCult.Caching
public string? DerivedFrom { get; }
}
// A record the id rule refuses: a duplicate id in one list, a random id that is not 12 lowercase hex characters, or a derived
// id whose source member is null or empty. It names the record, the list (record key, member slot, dictionary key and
// element index) and, where there is one, the element id and the source member.
public sealed class CultElementIdException : InvalidOperationException
{
public CultElementIdException(string message, string recordKey, string listPath, string? elementId = null, string? member = null)
: base(message)
{
RecordKey = recordKey;
ListPath = listPath;
ElementId = elementId;
Member = member;
}
public string RecordKey { get; }
public string ListPath { get; }
public string? ElementId { get; }
public string? Member { get; }
}
// The one owner of element identity: which types need an id, and how unset ids are filled.
// An id is 12 lowercase hex characters (48 bits); it need only be unique within one list. A write mints a random id; a
// load of a store written before ids existed mints from SHA-256(record key, element path), so every reader of the same
// store mints the same id and a reload is identical to what was read.
// A random id is 12 lowercase hex characters (48 bits); a derived id is the invariant text of its source and only has to be
// non-empty. Either need only be unique within one list. A write mints a random id; a load of a store written before ids
// existed mints from SHA-256(record key, element path), so every reader of the same store mints the same id and a reload is
// identical to what was read. A write and a load refuse the same things.
internal static class CultElementIds
{
private const int IdBytes = 6;
private sealed class Shape
// The ids a pass would fill, decided before anything is written: nothing changes until Apply, and Undo restores what
// Apply replaced.
internal sealed class IdPlan
{
private readonly List<(object Item, Shape Shape, string? Before, string Id)> _fills = new();
public int Count => _fills.Count;
internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id));
public void Apply()
{
foreach (var (item, shape, _, id) in _fills)
shape.SetId!(item, id);
}
public void Undo()
{
foreach (var (item, shape, before, _) in _fills)
shape.SetId!(item, before!);
}
}
internal sealed class Shape
{
public Func<object, string?>? GetId;
public Action<object, string>? SetId;
public Func<object, string>? Derive;
public Func<object, string?>? Derive;
public string? DerivedFrom;
public (int Slot, Func<object, object?> Get)[] Walk = Array.Empty<(int, Func<object, object?>)>();
}
@ -46,32 +92,41 @@ namespace GameCult.Caching
// ---- registration: which element types must carry an id ----
// Every reason this document type cannot be registered under the id rule, one per offending element type.
internal static List<string> Problems(Type documentType)
internal sealed class Survey
{
var problems = new List<string>();
VisitObject(documentType, documentType, new HashSet<Type>(), problems);
return problems;
public List<string> Problems { get; } = new();
// True when the type reaches an object list: what it writes can hold element ids.
public bool CarriesIds { get; set; }
}
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, List<string> problems)
// Every reason this document type cannot be registered under the id rule, one per offending concrete element type,
// and whether it reaches any element list at all.
internal static Survey Inspect(Type documentType)
{
var survey = new Survey();
VisitObject(documentType, documentType, new HashSet<Type>(), survey);
return survey;
}
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, Survey survey)
{
if (!seen.Add(type))
return;
foreach (var (member, memberType) in KeyedMembers(type))
VisitMemberType(documentType, type, member, memberType, seen, problems);
VisitMemberType(documentType, type, member, memberType, seen, survey);
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
VisitObject(documentType, union.SubType, seen, problems);
VisitObject(documentType, union.SubType, seen, survey);
}
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet<Type> seen, List<string> problems)
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet<Type> seen, Survey survey)
{
memberType = Nullable.GetUnderlyingType(memberType) ?? memberType;
if (IsLeaf(memberType))
return;
if (DictionaryValueType(memberType) is { } valueType)
{
VisitMemberType(documentType, owner, member, valueType, seen, problems);
VisitMemberType(documentType, owner, member, valueType, seen, survey);
return;
}
@ -79,18 +134,44 @@ namespace GameCult.Caching
{
if (IsObjectType(element))
{
var problem = IdProblem(element);
if (problem != null)
problems.Add($"Cult document {documentType.Name}: list element type {element.Name} (in {owner.Name}.{member.Name}) {problem}");
VisitObject(documentType, element, seen, problems);
survey.CarriesIds = true;
foreach (var concrete in ConcreteTypes(element))
{
var problem = IdProblem(concrete);
if (problem != null)
survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}");
}
VisitObject(documentType, element, seen, survey);
}
else
VisitMemberType(documentType, owner, member, element, seen, problems);
VisitMemberType(documentType, owner, member, element, seen, survey);
return;
}
if (IsObjectType(memberType))
VisitObject(documentType, memberType, seen, problems);
VisitObject(documentType, memberType, seen, survey);
}
// The types an element can actually be: the element type itself when it can be instantiated, and every union subtype
// under it. Shapes are built from these, so registration judges exactly what minting will meet.
private static IEnumerable<Type> ConcreteTypes(Type element)
{
var seen = new HashSet<Type>();
return Expand(element).ToList();
IEnumerable<Type> Expand(Type type)
{
if (!seen.Add(type))
yield break;
if (!type.IsAbstract && !type.IsInterface)
yield return type;
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
{
foreach (var subtype in Expand(union.SubType))
yield return subtype;
}
}
}
private static string? IdProblem(Type element)
@ -103,8 +184,11 @@ namespace GameCult.Caching
var id = ids[0];
if (TypeOf(id) != typeof(string))
return $"declares [CultElementId] on {id.Name}, which is not a string.";
if (KeyOf(id) == null)
if (KeyOf(id) is not { } idKey)
return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted.";
var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey);
if (sharing.Member != null)
return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own.";
if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true })
return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it.";
var source = id.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom;
@ -123,13 +207,14 @@ namespace GameCult.Caching
// ---- write and load: mint and check ----
// Fills every unset element id under root and returns how many were unset. Writes refuse a duplicate id within one
// list. dryRun counts and changes nothing. rootPath (the record key) seeds deterministic minting.
internal static int Assign(object root, string rootPath, bool deterministic, bool refuseDuplicates, bool dryRun = false)
// Decides every unset element id under root, and refuses what the rule refuses, without changing anything: a random id
// that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty.
// rootPath (the record key) seeds deterministic minting and names the record in a refusal.
internal static IdPlan Plan(object? root, string rootPath, bool deterministic)
{
var count = 0;
var plan = new IdPlan();
Visit(root, rootPath);
return count;
return plan;
void Visit(object? value, string path)
{
@ -141,14 +226,14 @@ namespace GameCult.Caching
if (value is IDictionary dictionary)
{
foreach (DictionaryEntry entry in dictionary)
Visit(entry.Value, path + "{" + entry.Key + "}");
Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}");
}
else if (value is IEnumerable list)
Elements(list, path);
else if (IsObjectType(type))
{
foreach (var (slot, get) in ShapeOf(type).Walk)
Visit(get(value), path + "." + slot);
Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture));
}
}
@ -158,10 +243,17 @@ namespace GameCult.Caching
var taken = new HashSet<string>(StringComparer.Ordinal);
foreach (var item in items)
{
var id = item != null && IsObjectType(item.GetType()) ? ShapeOf(item.GetType()).GetId?.Invoke(item) : null;
if (!string.IsNullOrEmpty(id) && !taken.Add(id!) && refuseDuplicates)
throw new InvalidOperationException(
$"Element id '{id}' appears twice in one list ({path}); an element id is unique within its list.");
if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape)
continue;
var id = getId(item);
if (string.IsNullOrEmpty(id))
continue;
if (shape.Derive == null && !IsRandomId(id!))
throw new CultElementIdException(
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters.",
rootPath, path, id);
if (!taken.Add(id!))
throw Duplicate(id!, path);
}
for (var index = 0; index < items.Length; index++)
@ -169,26 +261,46 @@ namespace GameCult.Caching
var item = items[index];
if (item == null)
continue;
var itemPath = path + "/" + index;
if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } set, GetId: { } get } itemShape)
var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture);
if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape &&
string.IsNullOrEmpty(get(item)))
{
if (string.IsNullOrEmpty(get(item)))
string id;
if (itemShape.Derive != null)
{
count++;
if (!dryRun)
{
var id = itemShape.Derive?.Invoke(item) ?? Mint(itemPath, deterministic, rootPath, taken);
if (itemShape.Derive != null && !taken.Add(id) && refuseDuplicates)
throw new InvalidOperationException(
$"Element id '{id}' appears twice in one list ({path}); an element id is unique within its list.");
set(item, id);
}
id = itemShape.Derive(item) ?? string.Empty;
if (id.Length == 0)
throw new CultElementIdException(
$"Record {rootPath}: the element at {itemPath} derives its id from {itemShape.DerivedFrom}, which is null or empty.",
rootPath, itemPath, member: itemShape.DerivedFrom);
if (!taken.Add(id))
throw Duplicate(id, path);
}
else
id = Mint(itemPath, deterministic, rootPath, taken);
plan.Add(item, itemShape, get(item), id);
}
Visit(item, itemPath);
}
}
CultElementIdException Duplicate(string id, string path) => new(
$"Record {rootPath}: element id '{id}' appears twice in the list at {path}; an element id is unique within its list.",
rootPath, path, id);
}
private static bool IsRandomId(string id)
{
if (id.Length != IdBytes * 2)
return false;
foreach (var character in id)
{
if (!(character is >= '0' and <= '9' or >= 'a' and <= 'f'))
return false;
}
return true;
}
private static string Mint(string itemPath, bool deterministic, string rootPath, HashSet<string> taken)
@ -199,7 +311,7 @@ namespace GameCult.Caching
if (deterministic)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt));
var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt.ToString(CultureInfo.InvariantCulture)));
Array.Copy(hash, bytes, IdBytes);
}
else
@ -220,7 +332,7 @@ namespace GameCult.Caching
{
var text = new StringBuilder(bytes.Length * 2);
foreach (var value in bytes)
text.Append(value.ToString("x2"));
text.Append(value.ToString("x2", CultureInfo.InvariantCulture));
return text.ToString();
}
@ -238,7 +350,10 @@ namespace GameCult.Caching
shape.SetId = idMember is FieldInfo f ? (o, v) => f.SetValue(o, v) : (o, v) => ((PropertyInfo)idMember).SetValue(o, v);
if (idMember.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom is { } source &&
MembersOf(type).FirstOrDefault(member => member.Name == source) is { } from)
shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty;
{
shape.DerivedFrom = source;
shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), CultureInfo.InvariantCulture);
}
}
shape.Walk = KeyedMembers(type)