diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index ef1b4092..d7ed650e 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -185,7 +185,7 @@ public static class CultDocumentMessagePackSerialization if (fieldCount > 0) { snapshot.FormatVersion = reader.ReadString() - ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + "."); + ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); } if (fieldCount > 1) @@ -221,20 +221,21 @@ public static class CultDocumentMessagePackSerialization { var version = snapshot.FormatVersion; if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) && - !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal)) + !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal) && + !string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal)) { throw new NotSupportedException( - $"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}."); + $"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}."); } - // Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust. + // Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust. if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal)) { var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null); if (variant != null) { throw new NotSupportedException( - $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2}."); + $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2} or {CultPersistedStoreSnapshot.FormatV3}."); } } } diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 8244d76a..e18efe70 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -16,6 +16,9 @@ namespace GameCult.Caching.MessagePack; public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { private const string IndexedFormatVersion = "cultcache.store.v4.directory-content-addressed-pages"; + // The manifest of a directory that holds element ids: a reader older than ids refuses it, because it would skip the id + // slots of a page and rewrite the elements without them. + private const string IndexedFormatVersionWithIds = "cultcache.store.v5.directory-content-addressed-pages"; // An unleased load that has not settled after this many attempts throws. private const int UnleasedLoadAttempts = 5; private readonly FileInfo _manifestFile; @@ -283,9 +286,12 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore pagePayload); } + // A manifest already marked stays marked: pages it names may hold ids this cache cannot read. + var carriesIds = string.Equals(currentManifest.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal) || + keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.Descriptor.CarriesElementIds); WriteManifest(targetCatalog, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray()); + .ToArray(), carriesIds); DeleteUnreferencedRecordPages(currentIndex.Values); @@ -295,11 +301,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore MarkFlushSucceeded(); } - private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index) + private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, bool carriesElementIds) { var manifest = new CultPersistedStoreSnapshot { - FormatVersion = IndexedFormatVersion, + FormatVersion = carriesElementIds ? IndexedFormatVersionWithIds : IndexedFormatVersion, SchemaCatalog = catalog, Records = index }; @@ -444,10 +450,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore } var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes); - if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal)) + if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) && + !string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal)) { throw new InvalidOperationException( - $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} is readable."); + $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable."); } return snapshot; diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 7b467c33..cbfb3d57 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -64,6 +64,7 @@ namespace GameCult.Caching { public const string FormatV1 = "cultcache.store.v1"; public const string FormatV2 = "cultcache.store.v2"; + public const string FormatV3 = "cultcache.store.v3"; public string FormatVersion { get; set; } = FormatV1; public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty(); @@ -210,6 +211,9 @@ namespace GameCult.Caching internal IReadOnlyList Members { get; } internal IReadOnlyList RichMembers { get; } + // True when the type reaches an object list: what it writes can hold element ids, so a store holding it carries the id marker. + public bool CarriesElementIds { get; internal set; } + // The public read surface (CultNet selection cut 1, section 6): declared addressing, and the // three ways to read a value through it. GameCult.Networking has no InternalsVisibleTo here on // purpose - this is the only door, so nothing outside the cache decides what a member means. @@ -376,9 +380,11 @@ namespace GameCult.Caching // A plain record staged by Flatten: the one way a plain record may replace a variant at its key. internal bool Flattens { get; set; } - internal bool IsResolved => _document != null; + // True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds). + internal bool IdsInMemoryOnly { get; set; } - internal CultStoredDocument Resolved(object document) => new(Key, StoredAt, Descriptor, Variant!, document); + internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly) => + new(Key, StoredAt, Descriptor, delta, document) { IdsInMemoryOnly = idsInMemoryOnly }; } public sealed class CultDocumentRegistry @@ -632,8 +638,9 @@ namespace GameCult.Caching ?? throw new InvalidOperationException( $"Type {type.FullName} is not marked with {nameof(CultDocumentAttribute)}."); var members = DiscoverMembers(type); - if (CultElementIds.Problems(type) is { Count: > 0 } idProblems) - throw new InvalidOperationException(idProblems[0]); + var ids = CultElementIds.Inspect(type); + if (ids.Problems.Count > 0) + throw new InvalidOperationException(ids.Problems[0]); var nameMember = members.FirstOrDefault(member => member.IsName); var indexAccessors = members .Where(member => member.IndexAlias != null) @@ -671,7 +678,10 @@ namespace GameCult.Caching nameMember?.GetterNullable, indexAccessors, descriptorMembers, - members); + members) + { + CarriesElementIds = ids.CarriesIds + }; } private static string BuildSemanticFingerprint( @@ -1800,8 +1810,6 @@ namespace GameCult.Caching private readonly CultDocumentRegistry _registry; private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); - // Keys whose element ids were minted at load and not yet written (see MintElementIds). - private readonly HashSet _mintedOnLoad = new(StringComparer.Ordinal); private readonly Dictionary> _names = new(); // Every key holding a value (unordered): the index is truthful about duplicates, so removing one holder never hides another. private readonly Dictionary<(Type Type, string Alias), Dictionary>> _indexes = new(); @@ -1830,6 +1838,19 @@ namespace GameCult.Caching // Read under the gate, by a store judging a merge. internal bool HoldsVariants => _variantKeys.Count > 0; + // A store wrote its whole view: a plain record it wrote carries the ids it held, so it is no longer in-memory only. A + // variant is not cleared here: the store wrote the delta it was handed, and only a write of the variant changes that. + internal void IdsPersisted(IEnumerable keys) => Held(() => + { + foreach (var key in keys) + { + if (_entries.TryGetValue(key, out var entry) && entry.Variant == null) + entry.IdsInMemoryOnly = false; + } + + return true; + }); + public CultCodec? Codec => _codec; // The whole override of one member, encoded with this cache's codec, ready for UpsertVariant. @@ -1838,8 +1859,6 @@ namespace GameCult.Caching if (_codec == null) throw new InvalidOperationException("This cache has no codec; create it with CultCacheMessagePack.Create."); var found = _registry.GetRequired(documentType).RichMembers.FirstOrDefault(entry => entry.Member.Name == member) ?? throw new ArgumentException($"{documentType.Name} has no persisted member '{member}'.", nameof(member)); - if (value != null) - CultElementIds.Assign(value, string.Empty, deterministic: false, refuseDuplicates: true); return CultVariantOverride.Set( found.Slot, _codec.Serialize(value!, found.MemberType, documentType)); @@ -1852,18 +1871,22 @@ namespace GameCult.Caching // it rewrote; a second call finds none. public int MintElementIds() { - var groups = Held(() => _mintedOnLoad - .Select(key => _entries.TryGetValue(key, out var entry) ? entry : null) - .Where(entry => entry is { Variant: null }) - .GroupBy(entry => Home(entry!.Descriptor.DocumentType)) - .Select(group => group.Select(entry => entry!).ToArray()) + var groups = Held(() => _entries.Values + .Where(entry => entry.IdsInMemoryOnly) + .GroupBy(entry => Home(entry.Descriptor.DocumentType)) + .Select(group => group.ToArray()) .ToArray()); foreach (var group in groups) { Commit(batch => { foreach (var entry in group) - batch.Upsert(entry.Descriptor.DocumentType, entry.Document, entry.Key); + { + if (entry.Variant is { } delta) + batch.UpsertVariant(entry.Key, new CultRecordKey(delta.BaseKey), delta.Overrides); + else + batch.Upsert(entry.Descriptor.DocumentType, entry.Document, entry.Key); + } }); } @@ -2348,30 +2371,47 @@ namespace GameCult.Caching if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); var home = Validate(admitted, evicted, source); - // Ids are filled before anything lands: a write mints random ids and refuses a duplicate in one list; a load of a - // record written before ids existed mints the ids its reload will mint again, and lands in memory only. - var mintedOnLoad = new List(); - foreach (var stored in admitted) + // Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that + // refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back. + // A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint + // again, and lands in memory only (variants mint inside Resolve, into their override values). + var idPlans = admitted + .Where(stored => stored.Variant == null) + .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Document, stored.Key.Value, deterministic: source != null))) + .ToArray(); + VariantPlan plan; + CultCommitOutcome outcome; + try { - if (stored.Variant == null && stored.IsResolved && - CultElementIds.Assign(stored.Document, stored.Key.Value, deterministic: source != null, refuseDuplicates: source == null) > 0 && - source != null) - mintedOnLoad.Add(stored.Key.Value); + foreach (var (_, idPlan) in idPlans) + idPlan.Apply(); + // The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch + // set, before land and before memory. land receives the admitted records with variants resolved. + plan = Resolve(admitted, evicted, source); + outcome = land(home, plan.Admitted); + } + catch + { + foreach (var (_, idPlan) in idPlans) + idPlan.Undo(); + throw; } - // The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch - // set, before land and before memory. land receives the admitted records with variants resolved. - var plan = Resolve(admitted, evicted, source); - var outcome = land(home, plan.Admitted); if (outcome != CultCommitOutcome.Committed) + { + foreach (var (_, idPlan) in idPlans) + idPlan.Undo(); return outcome; - // A record whose ids exist only in memory stays listed until a write persists them or the record goes. - foreach (var stored in admitted) - _mintedOnLoad.Remove(stored.Key.Value); - foreach (var stored in evicted) - _mintedOnLoad.Remove(stored.Key.Value); - foreach (var key in mintedOnLoad) - _mintedOnLoad.Add(key); + } + + // A record whose ids exist only in memory stays marked until a write persists them (a flush of the whole store + // does, and so does any write of the record: a new entry is unmarked). + if (source != null) + { + foreach (var (stored, idPlan) in idPlans) + stored.IdsInMemoryOnly = idPlan.Count > 0; + } + foreach (var change in Apply(plan, evicted, source)) _held.Add((change, source != null)); return CultCommitOutcome.Committed; @@ -2623,6 +2663,8 @@ namespace GameCult.Caching var type = descriptor.DocumentType; var kept = new List>(); + var overrides = new List(delta.Overrides.Count); + var minted = false; var seen = new HashSet(); foreach (var entry in delta.Overrides) { @@ -2638,9 +2680,10 @@ namespace GameCult.Caching if (member == null) throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have."); + object? value; try { - _codec.Deserialize(member.MemberType, type, entry.Value); + value = _codec.Deserialize(member.MemberType, type, entry.Value); } catch (Exception exception) { @@ -2650,7 +2693,19 @@ namespace GameCult.Caching exception.GetBaseException().Message, exception); } - kept.Add(new KeyValuePair(slot, entry.Value)); + // The override's value is minted like a plain record's members: ids the store never held are filled here, in the + // decoded copy, and the delta this hold lands carries them. + var bytes = entry.Value; + var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); + if (idPlan.Count > 0) + { + idPlan.Apply(); + bytes = _codec.Serialize(value!, member.MemberType, type); + minted = true; + } + + overrides.Add(new CultVariantOverride(entry.Op, entry.Path, entry.Id, bytes)); + kept.Add(new KeyValuePair(slot, bytes)); } var payload = _codec.Serialize(baseStored.Document, type, type); @@ -2674,7 +2729,7 @@ namespace GameCult.Caching stack.RemoveAt(stack.Count - 1); return done[key] = isVariant - ? variant.Resolved(document) + ? variant.Resolved(document, minted ? new CultVariantDelta(delta.BaseKey, overrides) : delta, variant.IdsInMemoryOnly || (minted && source != null)) : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { Flattens = true }; } } @@ -3168,7 +3223,9 @@ namespace GameCult.Caching { WriteSnapshot( Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())); + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()), + Entries.Values.Any(entry => entry.Descriptor.CarriesElementIds)); + Cache?.IdsPersisted(Entries.Keys); } MarkFlushSucceeded(); @@ -3210,7 +3267,14 @@ namespace GameCult.Caching foreach (var entry in request.Upserts) records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); - WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry()))); + // A file already marked stays marked: it may hold records this cache cannot read, and a rewrite must not shed the marker. + var carriesIds = request.Upserts.Any(entry => entry.Descriptor.CarriesElementIds) || + (ontoDisk + ? string.Equals(disk.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) + : Entries.Values.Any(entry => entry.Descriptor.CarriesElementIds)); + WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), carriesIds); + if (!ontoDisk) + Cache?.IdsPersisted(records.Keys); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) @@ -3263,16 +3327,20 @@ namespace GameCult.Caching } } - private void WriteSnapshot(IEnumerable records, IEnumerable catalog) + private void WriteSnapshot(IEnumerable records, IEnumerable catalog, bool carriesElementIds) { var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { - // v2 exactly when the store holds a variant: a store without one stays byte-identical v1. - FormatVersion = ordered.Any(record => record.Variant != null) - ? CultPersistedStoreSnapshot.FormatV2 - : CultPersistedStoreSnapshot.FormatV1, + // v3 when the store can hold element ids: a reader older than ids refuses the header, because it would skip the id + // slots and rewrite the elements without them. Otherwise v2 exactly when the store holds a variant, and a store + // with neither stays byte-identical v1. + FormatVersion = carriesElementIds + ? CultPersistedStoreSnapshot.FormatV3 + : ordered.Any(record => record.Variant != null) + ? CultPersistedStoreSnapshot.FormatV2 + : CultPersistedStoreSnapshot.FormatV1, SchemaCatalog = catalog .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) .Select(group => group.First()) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 28ad0836..c1991ba0 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -2,6 +2,7 @@ using System; using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Reflection; using System.Security.Cryptography; @@ -11,7 +12,7 @@ using MessagePack; namespace GameCult.Caching { // The identity of one element of an object list. A string member, keyed like any other, on every object type that - // sits in a list of a registered document type (union bases included: subclasses inherit the member). + // sits in a list of a registered document type (union bases included: every concrete subtype carries the member). // [CultElementId(nameof(Offset))] declares the id derived, never random: an unset id becomes the invariant-culture text of // that member (a keyed string or number on the same type, unique within its list). Content-addressed elements use it so the // same content always encodes to the same bytes. @@ -26,19 +27,64 @@ namespace GameCult.Caching public string? DerivedFrom { get; } } + // A record the id rule refuses: a duplicate id in one list, a random id that is not 12 lowercase hex characters, or a derived + // id whose source member is null or empty. It names the record, the list (record key, member slot, dictionary key and + // element index) and, where there is one, the element id and the source member. + public sealed class CultElementIdException : InvalidOperationException + { + public CultElementIdException(string message, string recordKey, string listPath, string? elementId = null, string? member = null) + : base(message) + { + RecordKey = recordKey; + ListPath = listPath; + ElementId = elementId; + Member = member; + } + + public string RecordKey { get; } + public string ListPath { get; } + public string? ElementId { get; } + public string? Member { get; } + } + // The one owner of element identity: which types need an id, and how unset ids are filled. - // An id is 12 lowercase hex characters (48 bits); it need only be unique within one list. A write mints a random id; a - // load of a store written before ids existed mints from SHA-256(record key, element path), so every reader of the same - // store mints the same id and a reload is identical to what was read. + // A random id is 12 lowercase hex characters (48 bits); a derived id is the invariant text of its source and only has to be + // non-empty. Either need only be unique within one list. A write mints a random id; a load of a store written before ids + // existed mints from SHA-256(record key, element path), so every reader of the same store mints the same id and a reload is + // identical to what was read. A write and a load refuse the same things. internal static class CultElementIds { private const int IdBytes = 6; - private sealed class Shape + // The ids a pass would fill, decided before anything is written: nothing changes until Apply, and Undo restores what + // Apply replaced. + internal sealed class IdPlan + { + private readonly List<(object Item, Shape Shape, string? Before, string Id)> _fills = new(); + + public int Count => _fills.Count; + + internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id)); + + public void Apply() + { + foreach (var (item, shape, _, id) in _fills) + shape.SetId!(item, id); + } + + public void Undo() + { + foreach (var (item, shape, before, _) in _fills) + shape.SetId!(item, before!); + } + } + + internal sealed class Shape { public Func? GetId; public Action? SetId; - public Func? Derive; + public Func? Derive; + public string? DerivedFrom; public (int Slot, Func Get)[] Walk = Array.Empty<(int, Func)>(); } @@ -46,32 +92,41 @@ namespace GameCult.Caching // ---- registration: which element types must carry an id ---- - // Every reason this document type cannot be registered under the id rule, one per offending element type. - internal static List Problems(Type documentType) + internal sealed class Survey { - var problems = new List(); - VisitObject(documentType, documentType, new HashSet(), problems); - return problems; + public List Problems { get; } = new(); + + // True when the type reaches an object list: what it writes can hold element ids. + public bool CarriesIds { get; set; } } - private static void VisitObject(Type documentType, Type type, HashSet seen, List problems) + // Every reason this document type cannot be registered under the id rule, one per offending concrete element type, + // and whether it reaches any element list at all. + internal static Survey Inspect(Type documentType) + { + var survey = new Survey(); + VisitObject(documentType, documentType, new HashSet(), survey); + return survey; + } + + private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey) { if (!seen.Add(type)) return; foreach (var (member, memberType) in KeyedMembers(type)) - VisitMemberType(documentType, type, member, memberType, seen, problems); + VisitMemberType(documentType, type, member, memberType, seen, survey); foreach (var union in type.GetCustomAttributes(true)) - VisitObject(documentType, union.SubType, seen, problems); + VisitObject(documentType, union.SubType, seen, survey); } - private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, List problems) + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, HashSet seen, Survey survey) { memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; if (IsLeaf(memberType)) return; if (DictionaryValueType(memberType) is { } valueType) { - VisitMemberType(documentType, owner, member, valueType, seen, problems); + VisitMemberType(documentType, owner, member, valueType, seen, survey); return; } @@ -79,18 +134,44 @@ namespace GameCult.Caching { if (IsObjectType(element)) { - var problem = IdProblem(element); - if (problem != null) - problems.Add($"Cult document {documentType.Name}: list element type {element.Name} (in {owner.Name}.{member.Name}) {problem}"); - VisitObject(documentType, element, seen, problems); + survey.CarriesIds = true; + foreach (var concrete in ConcreteTypes(element)) + { + var problem = IdProblem(concrete); + if (problem != null) + survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); + } + + VisitObject(documentType, element, seen, survey); } else - VisitMemberType(documentType, owner, member, element, seen, problems); + VisitMemberType(documentType, owner, member, element, seen, survey); return; } if (IsObjectType(memberType)) - VisitObject(documentType, memberType, seen, problems); + VisitObject(documentType, memberType, seen, survey); + } + + // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype + // under it. Shapes are built from these, so registration judges exactly what minting will meet. + private static IEnumerable ConcreteTypes(Type element) + { + var seen = new HashSet(); + return Expand(element).ToList(); + + IEnumerable Expand(Type type) + { + if (!seen.Add(type)) + yield break; + if (!type.IsAbstract && !type.IsInterface) + yield return type; + foreach (var union in type.GetCustomAttributes(true)) + { + foreach (var subtype in Expand(union.SubType)) + yield return subtype; + } + } } private static string? IdProblem(Type element) @@ -103,8 +184,11 @@ namespace GameCult.Caching var id = ids[0]; if (TypeOf(id) != typeof(string)) return $"declares [CultElementId] on {id.Name}, which is not a string."; - if (KeyOf(id) == null) + if (KeyOf(id) is not { } idKey) return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted."; + var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey); + if (sharing.Member != null) + return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own."; if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true }) return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it."; var source = id.GetCustomAttribute(true)!.DerivedFrom; @@ -123,13 +207,14 @@ namespace GameCult.Caching // ---- write and load: mint and check ---- - // Fills every unset element id under root and returns how many were unset. Writes refuse a duplicate id within one - // list. dryRun counts and changes nothing. rootPath (the record key) seeds deterministic minting. - internal static int Assign(object root, string rootPath, bool deterministic, bool refuseDuplicates, bool dryRun = false) + // Decides every unset element id under root, and refuses what the rule refuses, without changing anything: a random id + // that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty. + // rootPath (the record key) seeds deterministic minting and names the record in a refusal. + internal static IdPlan Plan(object? root, string rootPath, bool deterministic) { - var count = 0; + var plan = new IdPlan(); Visit(root, rootPath); - return count; + return plan; void Visit(object? value, string path) { @@ -141,14 +226,14 @@ namespace GameCult.Caching if (value is IDictionary dictionary) { foreach (DictionaryEntry entry in dictionary) - Visit(entry.Value, path + "{" + entry.Key + "}"); + Visit(entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); } else if (value is IEnumerable list) Elements(list, path); else if (IsObjectType(type)) { foreach (var (slot, get) in ShapeOf(type).Walk) - Visit(get(value), path + "." + slot); + Visit(get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); } } @@ -158,10 +243,17 @@ namespace GameCult.Caching var taken = new HashSet(StringComparer.Ordinal); foreach (var item in items) { - var id = item != null && IsObjectType(item.GetType()) ? ShapeOf(item.GetType()).GetId?.Invoke(item) : null; - if (!string.IsNullOrEmpty(id) && !taken.Add(id!) && refuseDuplicates) - throw new InvalidOperationException( - $"Element id '{id}' appears twice in one list ({path}); an element id is unique within its list."); + if (item == null || !IsObjectType(item.GetType()) || ShapeOf(item.GetType()) is not { GetId: { } getId } shape) + continue; + var id = getId(item); + if (string.IsNullOrEmpty(id)) + continue; + if (shape.Derive == null && !IsRandomId(id!)) + throw new CultElementIdException( + $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters.", + rootPath, path, id); + if (!taken.Add(id!)) + throw Duplicate(id!, path); } for (var index = 0; index < items.Length; index++) @@ -169,26 +261,46 @@ namespace GameCult.Caching var item = items[index]; if (item == null) continue; - var itemPath = path + "/" + index; - if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } set, GetId: { } get } itemShape) + var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture); + if (IsObjectType(item.GetType()) && ShapeOf(item.GetType()) is { SetId: { } , GetId: { } get } itemShape && + string.IsNullOrEmpty(get(item))) { - if (string.IsNullOrEmpty(get(item))) + string id; + if (itemShape.Derive != null) { - count++; - if (!dryRun) - { - var id = itemShape.Derive?.Invoke(item) ?? Mint(itemPath, deterministic, rootPath, taken); - if (itemShape.Derive != null && !taken.Add(id) && refuseDuplicates) - throw new InvalidOperationException( - $"Element id '{id}' appears twice in one list ({path}); an element id is unique within its list."); - set(item, id); - } + id = itemShape.Derive(item) ?? string.Empty; + if (id.Length == 0) + throw new CultElementIdException( + $"Record {rootPath}: the element at {itemPath} derives its id from {itemShape.DerivedFrom}, which is null or empty.", + rootPath, itemPath, member: itemShape.DerivedFrom); + if (!taken.Add(id)) + throw Duplicate(id, path); } + else + id = Mint(itemPath, deterministic, rootPath, taken); + plan.Add(item, itemShape, get(item), id); } Visit(item, itemPath); } } + + CultElementIdException Duplicate(string id, string path) => new( + $"Record {rootPath}: element id '{id}' appears twice in the list at {path}; an element id is unique within its list.", + rootPath, path, id); + } + + private static bool IsRandomId(string id) + { + if (id.Length != IdBytes * 2) + return false; + foreach (var character in id) + { + if (!(character is >= '0' and <= '9' or >= 'a' and <= 'f')) + return false; + } + + return true; } private static string Mint(string itemPath, bool deterministic, string rootPath, HashSet taken) @@ -199,7 +311,7 @@ namespace GameCult.Caching if (deterministic) { using var sha = SHA256.Create(); - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt)); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt.ToString(CultureInfo.InvariantCulture))); Array.Copy(hash, bytes, IdBytes); } else @@ -220,7 +332,7 @@ namespace GameCult.Caching { var text = new StringBuilder(bytes.Length * 2); foreach (var value in bytes) - text.Append(value.ToString("x2")); + text.Append(value.ToString("x2", CultureInfo.InvariantCulture)); return text.ToString(); } @@ -238,7 +350,10 @@ namespace GameCult.Caching shape.SetId = idMember is FieldInfo f ? (o, v) => f.SetValue(o, v) : (o, v) => ((PropertyInfo)idMember).SetValue(o, v); if (idMember.GetCustomAttribute(true)!.DerivedFrom is { } source && MembersOf(type).FirstOrDefault(member => member.Name == source) is { } from) - shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + { + shape.DerivedFrom = source; + shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), CultureInfo.InvariantCulture); + } } shape.Walk = KeyedMembers(type)