From dbc87b94471053da04445a72a7e830352e3b959c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:51:53 +0200 Subject: [PATCH 01/16] C# core: variant records, top-level Set overrides, resolution in the admitting hold A variant record stores [baseKey, overrides[]] in record slot 4 and an empty payload; the cache resolves it against its base inside the hold that admits it or any record on its base chain, and re-resolves dependents in the same Apply, so a base change reaches its variants with contiguous sequences and no stamp. Plain Upsert at a variant key is refused (Flatten is the explicit way); cycle, missing base, different concrete type, evicted base with survivors and a name that shadows the base are refused naming the keys. Overrides of a dropped slot or an undecodable value are ignored with a warning and reported as compatible drift. The store writes cultcache.store.v2 only when it holds a variant and reads v1 and v2. The codec pair CultInspectorModel took becomes CultCodec, shared with the cache, which needs it to clone documents and overlay override values. The directory store, CultMesh single-file reads and CultNet row building refuse a variant by name until their own cuts. Co-Authored-By: Claude Sonnet 5 --- docs/runtime-parity-scope.md | 3 + .../CultCacheMessagePack.cs | 21 +- .../CultDocumentMessagePackSerialization.cs | 160 +++++- .../DirectoryMessagePackBackingStore.cs | 11 + .../Contracts/cultcache-persistence-format.md | 36 +- .../cultcache-schema-compatibility.md | 7 + src/GameCult.Caching/CultCache.cs | 475 +++++++++++++++++- src/GameCult.Caching/CultInspectorModel.cs | 20 +- src/GameCult.Caching/CultVariants.cs | 95 ++++ .../CultMeshSingleFileDocuments.cs | 13 + .../CultNetDocumentRegistry.cs | 11 +- 11 files changed, 797 insertions(+), 55 deletions(-) create mode 100644 src/GameCult.Caching/CultVariants.cs diff --git a/docs/runtime-parity-scope.md b/docs/runtime-parity-scope.md index 46b1e77b..97fa1b48 100644 --- a/docs/runtime-parity-scope.md +++ b/docs/runtime-parity-scope.md @@ -15,6 +15,9 @@ Cross-runtime parity has three layers: without translation folklore. Every store reader refuses, by name, a header or record slot it does not understand (vectors: `tests/vectors/document-variants-c0`). + Document variants: C# resolves them (`cultcache.store.v2`); every other runtime, + CultMesh single-file reads, the directory store and CultNet refuse a variant by name + (vector written by C#: `tests/vectors/document-variants-c1`). - Feature parity: each runtime must expose the CultCache/CultNet/CultMesh surfaces needed for its intended role, with unsupported production-server ownership called out plainly. diff --git a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs index e1a68c12..4fa55076 100644 --- a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs +++ b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs @@ -19,13 +19,19 @@ namespace GameCult.Caching.MessagePack public static class CultCacheMessagePack { - // The inspection model over this codec. Values serialize without the document guard but with their owning document's - // options, resolvers included, so dictionary keys compare by the bytes the store writes; clones deserialize as - // registered documents. - public static CultInspectorModel CreateInspectorModel(CultDocumentRegistry registry) => new CultInspectorModel( - registry, + // The store's encoding as one port. Values serialize without the document guard but with their owning document's + // options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type + // deserializes as a registered document, a member type as itself. + public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec( (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), - (type, bytes) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry)); + (type, document, bytes) => type == document + ? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry) + : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + CultDocumentMessagePackSerialization.OverlaySlots); + + // The inspection model over that codec. + public static CultInspectorModel CreateInspectorModel(CultDocumentRegistry registry) => + new CultInspectorModel(registry, CreateCodec(registry)); // Opening hydrates. A consumer that wants a fresh store removes the records it does not keep, upserts, and // flushes; the atomic replace does the rest. @@ -37,7 +43,8 @@ namespace GameCult.Caching.MessagePack } options ??= new CultCacheOpenOptions(); - var cache = new CultCache(options.Registry) + var registry = options.Registry ?? CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CreateCodec(registry)) { FlushAttachedStoresOnDispose = options.FlushOnDispose }; diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index bb62d2ed..ef1b4092 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Reflection; using GameCult.Caching; @@ -30,8 +31,8 @@ public sealed class CultDocumentResolver : IFormatterResolver public static class CultDocumentMessagePackSerialization { - private const string SingleFileFormatVersion = "cultcache.store.v1"; private const int PersistedRecordFieldCount = 4; + private const int VariantRecordFieldCount = 5; private const int SchemaCatalogEntryFieldCount = 7; private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; @@ -95,6 +96,49 @@ public static class CultDocumentMessagePackSerialization ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } + /// + /// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member + /// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil. + /// + public static byte[] OverlaySlots(byte[] payload, IReadOnlyList> slots) + { + var reader = new MessagePackReader(payload); + if (reader.NextMessagePackType != MessagePackType.Array) + { + throw new InvalidOperationException("A document payload is a MessagePack array; this one is not, so its slots cannot be overridden."); + } + + var count = reader.ReadArrayHeader(); + var elements = new List(count); + for (var index = 0; index < count; index++) + { + var start = reader.Position; + reader.Skip(); + elements.Add(reader.Sequence.Slice(start, reader.Position).ToArray()); + } + + foreach (var (slot, value) in slots) + { + while (elements.Count <= slot) + { + elements.Add(new byte[] { 0xc0 }); + } + + elements[slot] = value; + } + + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(elements.Count); + foreach (var element in elements) + { + writer.WriteRaw(element); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + public static byte[] SerializePersistedRecord(CultPersistedRecord record) { var buffer = new global::System.Buffers.ArrayBufferWriter(); @@ -141,7 +185,7 @@ public static class CultDocumentMessagePackSerialization if (fieldCount > 0) { snapshot.FormatVersion = reader.ReadString() - ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + SingleFileFormatVersion + " only."); + ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + "."); } if (fieldCount > 1) @@ -172,24 +216,110 @@ public static class CultDocumentMessagePackSerialization return snapshot; } - /// Refuses a single-file snapshot this runtime cannot read, naming the format it found. + /// Refuses a single-file snapshot this runtime cannot read, naming the format or record it found. public static void RequireSingleFileFormat(CultPersistedStoreSnapshot snapshot) { - if (!string.Equals(snapshot.FormatVersion, SingleFileFormatVersion, StringComparison.Ordinal)) + var version = snapshot.FormatVersion; + if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) && + !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal)) { throw new NotSupportedException( - $"Store format {snapshot.FormatVersion} is not readable; this runtime reads {SingleFileFormatVersion} only. " + - "The store needs a runtime that resolves document variants."); + $"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}."); + } + + // Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust. + if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal)) + { + var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null); + if (variant != null) + { + throw new NotSupportedException( + $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2}."); + } } } private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) { - writer.WriteArrayHeader(PersistedRecordFieldCount); + writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount); writer.Write(record.Key); writer.Write(record.SchemaId); writer.Write(record.StoredAt); writer.Write(record.Payload); + if (record.Variant != null) + { + // Slot 4: [baseKey, overrides[]]; an override is [op, path[], id, value], a path step [slot, elementId], + // and the value is the member's own MessagePack encoding, inline. + writer.WriteArrayHeader(2); + writer.Write(record.Variant.BaseKey); + writer.WriteArrayHeader(record.Variant.Overrides.Count); + foreach (var entry in record.Variant.Overrides) + { + writer.WriteArrayHeader(4); + writer.Write((int)entry.Op); + writer.WriteArrayHeader(entry.Path.Count); + foreach (var step in entry.Path) + { + writer.WriteArrayHeader(2); + writer.Write(step.Slot); + writer.Write(step.ElementId); + } + + writer.Write(entry.Id); + writer.WriteRaw(entry.Value); + } + } + } + + private static CultVariantDelta ReadVariant(ref MessagePackReader reader, string key, string schemaId) + { + var slots = reader.ReadArrayHeader(); + if (slots != 2) + { + throw new NotSupportedException( + $"Record '{key}' (schema '{schemaId}') has a variant slot of {slots} entries; this runtime reads 2 (base key, overrides)."); + } + + var baseKey = reader.ReadString() ?? string.Empty; + var count = reader.ReadArrayHeader(); + var overrides = new CultVariantOverride[count]; + for (var index = 0; index < count; index++) + { + var fields = reader.ReadArrayHeader(); + if (fields != 4) + { + throw new NotSupportedException( + $"Record '{key}' (schema '{schemaId}') has an override of {fields} entries; this runtime reads 4 (op, path, id, value)."); + } + + var op = reader.ReadInt32(); + if (op is < 0 or > 2) + { + throw new NotSupportedException($"Record '{key}' (schema '{schemaId}') has an override with unknown op {op}."); + } + + var stepCount = reader.ReadArrayHeader(); + var path = new CultPathStep[stepCount]; + for (var step = 0; step < stepCount; step++) + { + var stepFields = reader.ReadArrayHeader(); + if (stepFields != 2) + { + throw new NotSupportedException( + $"Record '{key}' (schema '{schemaId}') has a path step of {stepFields} entries; this runtime reads 2 (slot, elementId)."); + } + + path[step] = new CultPathStep(reader.ReadInt32(), reader.ReadString() ?? string.Empty); + } + + var id = reader.ReadString() ?? string.Empty; + var start = reader.Position; + reader.Skip(); + var value = reader.Sequence.Slice(start, reader.Position).ToArray(); + overrides[index] = new CultVariantOverride((CultOverrideOp)op, path, id, value); + } + + return new CultVariantDelta(baseKey, overrides); } public static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader) @@ -217,11 +347,21 @@ public static class CultDocumentMessagePackSerialization record.Payload = reader.ReadBytes()?.ToArray() ?? Array.Empty(); } - if (fieldCount > PersistedRecordFieldCount) + if (fieldCount > VariantRecordFieldCount) { throw new NotSupportedException( - $"Record '{record.Key}' (schema '{record.SchemaId}') has {fieldCount} slots; this runtime reads {PersistedRecordFieldCount}. " + - "The store needs a runtime that resolves document variants."); + $"Record '{record.Key}' (schema '{record.SchemaId}') has {fieldCount} slots; this runtime reads {PersistedRecordFieldCount} " + + $"(or {VariantRecordFieldCount} for a variant)."); + } + + if (fieldCount == VariantRecordFieldCount) + { + record.Variant = ReadVariant(ref reader, record.Key, record.SchemaId); + if (record.Payload.Length != 0) + { + throw new NotSupportedException( + $"Record '{record.Key}' (schema '{record.SchemaId}') is a variant and must carry an empty payload."); + } } return record; diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 9e185202..fa9d4899 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -135,9 +135,18 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Trace("publish"); } + // Variants wait for the directory store's own cut; until then it refuses one rather than persist a resolved view as a plain page. + private static void RefuseVariant(CultStoredDocument entry) + { + if (entry.Variant != null) + throw new NotSupportedException( + $"Record '{entry.Key.Value}' is a variant of '{entry.Variant.BaseKey}'; the directory store does not hold document variants yet."); + } + public override void Push(CultStoredDocument entry) { ThrowIfReadOnly(); + RefuseVariant(entry); Held(() => { Entries[entry.Key.Value] = entry; @@ -162,6 +171,8 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) { ThrowIfReadOnly(); + foreach (var entry in request.Upserts) + RefuseVariant(entry); return Held(() => { Directory.CreateDirectory(_manifestFile.DirectoryName!); diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 4511f3c5..1205d904 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -213,9 +213,39 @@ The payload is the domain object only. It does not repeat the key, schema id, or stored timestamp. Under `cultcache.store.v1` a record has exactly these four slots. A reader refuses any -other `cultcache.store.*` header and any record with more than four slots, naming the -version or the record's key and schema id. It never skips a slot it does not -understand. +`cultcache.store.*` header it does not know and any record with more slots than its +header allows, naming the version or the record's key and schema id. It never skips a +slot it does not understand. + +## Variant Records + +A variant record is a record whose document is a base record's document with some +members replaced. It stores the difference, never the resolved document. It carries a +fifth slot: + +``` +[key, schemaId, storedAt, payload, [baseKey, overrides[]]] +``` + +- `payload` is empty. An older reader that ignores the fifth slot reads an empty + payload as a corrupt document and refuses, which is why the delta lives outside it. +- `schemaId` is the type's schema id, the same as the base's. A variant has its base's + concrete type, and lives in the same store (one home store per type). +- An override is `[op, path[], id, value]`. `op` is `0` Set, `1` Insert or `2` Remove. + A path step is `[slot, elementId]`: `slot` is the member's MessagePack `[Key]` and + `elementId` is `""` unless the step enters an object-list element. `value` is the + member's own MessagePack encoding, inline. A top-level Set has one step and `id` `""`; + a reader that resolves only that shape refuses the others, naming the variant. +- A store holding at least one variant declares `cultcache.store.v2`; a store holding + none declares `cultcache.store.v1` and is byte-identical to what a runtime without + variants writes. A `v1` header over a variant record is refused. +- A variant's resolved document is derived in memory when the cache admits the record + or any record on its base chain, and is never written back. A plain write at a + variant's key is refused; `Flatten` replaces a variant with a plain record holding its + resolved document. +- The C# runtime resolves variants. Every other runtime refuses a `v2` store by name. + CultMesh single-file reads, the directory store and CultNet refuse a variant record by + name until each has its own cut. ## Canonical Semantic Schema Hash diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index 297c3184..818840ee 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -105,3 +105,10 @@ lie about the payload: - no compatible local schema candidate This is not negotiable. Better a loud refusal than quiet bit-rot. + +## Variant overrides + +A variant's override drifts exactly as the same member would on a plain record: an +override naming a slot the type no longer has, or holding a value that no longer decodes +into the member's type, is ignored with a warning naming the variant and the member, and +the variant inherits the base value. There is no separate rule for variants. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2f8e316a..24c20cfc 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -55,11 +55,17 @@ namespace GameCult.Caching public string SchemaId { get; set; } = string.Empty; public string StoredAt { get; set; } = string.Empty; public byte[] Payload { get; set; } = Array.Empty(); + + // Non-null for a variant record (slot 4): its base and overrides. Its payload is empty. + public CultVariantDelta? Variant { get; set; } } public sealed class CultPersistedStoreSnapshot { - public string FormatVersion { get; set; } = "cultcache.store.v1"; + public const string FormatV1 = "cultcache.store.v1"; + public const string FormatV2 = "cultcache.store.v2"; + + public string FormatVersion { get; set; } = FormatV1; public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty(); public CultPersistedRecord[] Records { get; set; } = Array.Empty(); } @@ -336,13 +342,43 @@ namespace GameCult.Caching Key = key; StoredAt = storedAt; Descriptor = descriptor; - Document = document; + _document = document; } + // A variant: the delta is what is stored; the cache resolves it against its base in the hold that admits it. + internal CultStoredDocument( + CultRecordKey key, + string storedAt, + CultDocumentDescriptor descriptor, + CultVariantDelta variant, + object? resolved) + { + Key = key; + StoredAt = storedAt; + Descriptor = descriptor; + Variant = variant; + _document = resolved; + } + + private readonly object? _document; + public CultRecordKey Key { get; } public string StoredAt { get; } public CultDocumentDescriptor Descriptor { get; } - public object Document { get; } + + // The complete document. For a variant, the resolved view: derived, never written back. + public object Document => _document ?? + throw new InvalidOperationException($"Variant {Key.Value} has not been resolved; only the cache resolves a variant."); + + // The base and overrides a variant record stores; null for a plain record. + public CultVariantDelta? Variant { get; } + + // A plain record staged by Flatten: the one way a plain record may replace a variant at its key. + internal bool Flattens { get; set; } + + internal bool IsResolved => _document != null; + + internal CultStoredDocument Resolved(object document) => new(Key, StoredAt, Descriptor, Variant!, document); } public sealed class CultDocumentRegistry @@ -1662,11 +1698,29 @@ namespace GameCult.Caching private static string Identity(string key, string schemaId, string storedAt) => $"{key}\n{schemaId}\n{storedAt}"; } + // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. + internal sealed class StagedOperation + { + public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten) + { + Key = key; + Stored = stored; + Variant = variant; + Flatten = flatten; + } + + public CultRecordKey Key { get; } + public CultStoredDocument? Stored { get; } + public CultVariantDelta? Variant { get; } + public bool Flatten { get; } + public bool IsRemoval => Stored == null && Variant == null && !Flatten; + } + // An explicit value: nothing staged here is visible to anyone until Commit returns. public sealed class CultCacheBatch { private readonly CultCache _cache; - internal readonly Dictionary Operations = new(StringComparer.Ordinal); + internal readonly Dictionary Operations = new(StringComparer.Ordinal); internal readonly List<(CultRecordKey Key, string? SchemaId, string? StoredAt)> Expected = new(); internal bool ExpectsUnchanged; internal bool Sealed; @@ -1691,7 +1745,24 @@ namespace GameCult.Caching public void Remove(CultRecordKey key) { ThrowIfSealed(); - Operations[key.Value] = (key, null); + Operations[key.Value] = new StagedOperation(key, null, null, false); + } + + // Stages a variant record: its base and a sparse set of overrides, resolved by the cache when the batch commits. + // A variant changes only through here; rebasing is this call with another base. + public CultRecordKey UpsertVariant(CultRecordKey key, CultRecordKey baseKey, IEnumerable? overrides = null) + { + ThrowIfSealed(); + Operations[key.Value] = new StagedOperation( + key, null, new CultVariantDelta(baseKey.Value, (overrides ?? Array.Empty()).ToArray()), false); + return key; + } + + // Replaces a variant with a plain record holding its resolved document: the explicit, separate operation (Q1a). + public void Flatten(CultRecordKey key) + { + ThrowIfSealed(); + Operations[key.Value] = new StagedOperation(key, null, null, true); } public void Expect(CultRecordKey key, object? current) @@ -1711,7 +1782,7 @@ namespace GameCult.Caching { ThrowIfSealed(); var stored = _cache.CreateStoredDocument(document, key); - Operations[stored.Key.Value] = (stored.Key, stored); + Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false); return stored.Key; } @@ -1730,6 +1801,11 @@ namespace GameCult.Caching private readonly Dictionary> _names = new(); private readonly Dictionary<(Type Type, string Alias), Dictionary> _indexes = new(); private readonly Dictionary _globals = new(); + private readonly CultCodec? _codec; + // Keys holding a variant, and the drift report of each variant whose overrides were partly ignored: both derived + // from the entries by Apply, never decided anywhere else. + private readonly HashSet _variantKeys = new(StringComparer.Ordinal); + private readonly Dictionary _variantReports = new(StringComparer.Ordinal); private readonly ConditionalWeakTable _handles = new(); private readonly Subject _changes = new(); private readonly object _gate = new(); @@ -1738,13 +1814,42 @@ namespace GameCult.Caching [ThreadStatic] private static List<(Change Change, bool Loaded)>? _held; private bool _dirtyInMemory; - public CultCache(CultDocumentRegistry? registry = null) + // The codec is the store's encoding (CultCacheMessagePack.Create supplies it). A cache without one holds no variants: + // resolving a variant needs to clone documents and decode override values. + public CultCache(CultDocumentRegistry? registry = null, CultCodec? codec = null) { _registry = registry ?? CultDocumentRegistry.Shared; + _codec = codec; } public CultDocumentRegistry Registry => _registry; + public CultCodec? Codec => _codec; + + // Overrides a variant carries that no longer resolve (a slot the type dropped, a value that no longer decodes), + // reported as compatible drift: the override is ignored and the variant inherits the base value. + public IReadOnlyList VariantMigrationReports + { + get + { + lock (_gate) + return _variantReports.Values.ToArray(); + } + } + + // The whole override of one member, encoded with this cache's codec, ready for UpsertVariant. + public CultVariantOverride Override(Type documentType, string member, object? value) + { + if (_codec == null) throw new InvalidOperationException("This cache has no codec; create it with CultCacheMessagePack.Create."); + var found = _registry.GetRequired(documentType).RichMembers.FirstOrDefault(entry => entry.Member.Name == member) + ?? throw new ArgumentException($"{documentType.Name} has no persisted member '{member}'.", nameof(member)); + return CultVariantOverride.Set( + found.Slot, + _codec.Serialize(value!, found.MemberType, documentType)); + } + + public CultVariantOverride Override(string member, object? value) where T : class => Override(typeof(T), member, value); + public bool IsDirty { get @@ -1841,7 +1946,7 @@ namespace GameCult.Caching store.AttachRegistry(_registry); store.Cache = this; - store.Loaded = (loaded, dropped) => Admit(loaded, dropped, store, _ => CultCommitOutcome.Committed); + store.Loaded = (loaded, dropped) => Admit(loaded, dropped, store, (_, _) => CultCommitOutcome.Committed); _stores.Add(candidate); try { @@ -1915,11 +2020,39 @@ namespace GameCult.Caching return Task.FromResult(Write(document, key)); } + // A single-call variant write; a batch stages the same thing with CultCacheBatch.UpsertVariant. + public Task UpsertVariantAsync(CultRecordKey key, CultRecordKey baseKey, IEnumerable? overrides = null) + { + var delta = new CultVariantDelta(baseKey.Value, (overrides ?? Array.Empty()).ToArray()); + return Task.FromResult(Held(() => + { + var stored = StampVariant(key, DescriptorOfBase(key, delta.BaseKey, null), delta); + Admit(new[] { stored }, Array.Empty(), null, (home, docs) => + { + home?.Push(docs[0]); + return CultCommitOutcome.Committed; + }); + return key; + })); + } + + // Replaces a variant with a plain record holding its resolved document (Q1a's explicit operation). + public Task FlattenAsync(CultRecordKey key) => Task.FromResult(Held(() => + { + var stored = StampFlatten(key); + Admit(new[] { stored }, Array.Empty(), null, (home, docs) => + { + home?.Push(docs[0]); + return CultCommitOutcome.Committed; + }); + return key; + })); + public bool Remove(CultRecordKey key) => Held(() => { if (!_entries.TryGetValue(key.Value, out var existing)) return false; - Admit(Array.Empty(), new[] { existing }, null, home => + Admit(Array.Empty(), new[] { existing }, null, (home, _) => { home?.Delete(existing); return CultCommitOutcome.Committed; @@ -1950,6 +2083,13 @@ namespace GameCult.Caching return _handles.TryGetValue(document, out var box) ? new CultRecordHandle(box.Key) : null; } + // The stored form of a record: for a variant, the delta beside the resolved document. + public CultStoredDocument? GetStored(CultRecordKey key) + { + lock (_gate) + return _entries.TryGetValue(key.Value, out var stored) ? stored : null; + } + public object? Get(CultRecordKey key) { lock (_gate) @@ -2039,7 +2179,53 @@ namespace GameCult.Caching // Call under the gate that admits the result, so no load can land a later storedAt in between. private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document) => - new(key, MintStoredAt(_entries.TryGetValue(key.Value, out var previous) ? previous.StoredAt : null), descriptor, document); + new(key, MintStoredAt(PreviousStoredAt(key)), descriptor, document); + + // An unresolved variant: the admitting hold resolves it against its base before anything lands. + private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta) => + new(key, MintStoredAt(PreviousStoredAt(key)), descriptor, delta, null); + + // The plain record a Flatten stages: the variant's resolved document, under the variant's own key and schema. + private CultStoredDocument StampFlatten(CultRecordKey key) + { + if (!_entries.TryGetValue(key.Value, out var existing) || existing.Variant == null) + throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); + return new CultStoredDocument(key, MintStoredAt(existing.StoredAt), existing.Descriptor, existing.Document) { Flattens = true }; + } + + private string? PreviousStoredAt(CultRecordKey key) => _entries.TryGetValue(key.Value, out var previous) ? previous.StoredAt : null; + + // The concrete type a variant takes from its base, walking bases through what the batch stages and then the cache. + // Refuses a cycle or a base that is not there, naming the keys, exactly as the admitting hold would. + private CultDocumentDescriptor DescriptorOfBase(CultRecordKey variant, string baseKey, CultCacheBatch? batch) + { + var seen = new List { variant.Value }; + var next = baseKey; + while (true) + { + if (seen.Contains(next)) + throw CycleRefusal(seen.Append(next)); + var staged = batch != null && batch.Operations.TryGetValue(next, out var op) ? op : null; + if (staged != null) + { + if (staged.Stored != null) + return staged.Stored.Descriptor; + if (staged.Variant != null) + { + seen.Add(next); + next = staged.Variant.BaseKey; + continue; + } + + if (!staged.Flatten) + throw MissingBaseRefusal(variant.Value, baseKey, removed: true); + } + + if (!_entries.TryGetValue(next, out var existing)) + throw MissingBaseRefusal(variant.Value, baseKey, removed: false); + return existing.Descriptor; + } + } internal CultStoredDocument? Observe(CultRecordKey key, object? current) { @@ -2068,9 +2254,9 @@ namespace GameCult.Caching private CultRecordKey Write(object document, CultRecordKey? key) => Held(() => { var stored = CreateStoredDocument(document, key); - Admit(new[] { stored }, Array.Empty(), null, home => + Admit(new[] { stored }, Array.Empty(), null, (home, docs) => { - home?.Push(stored); + home?.Push(docs[0]); return CultCommitOutcome.Committed; }); return stored.Key; @@ -2094,16 +2280,19 @@ namespace GameCult.Caching { // Staging stamped outside this gate; a pull since then may hold a later storedAt, so stamp again. var upserts = batch.Operations.Values - .Where(op => op.Stored != null) - .Select(op => Stamp(op.Key, op.Stored!.Descriptor, op.Stored.Document)) + .Where(op => !op.IsRemoval) + .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document) + : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) + : StampFlatten(op.Key)) .ToArray(); var deletes = batch.Operations.Values - .Where(op => op.Stored == null && _entries.ContainsKey(op.Key.Value)) + .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) .Select(op => _entries[op.Key.Value]) .ToArray(); - var request = new CultCommitRequest(upserts, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); - return Admit(upserts, deletes, null, home => + return Admit(upserts, deletes, null, (home, resolved) => { + // What the store holds is what the hold resolved: a variant's delta beside its resolved view. + var request = new CultCommitRequest(resolved, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); if (home != null) { if (request.HasConditions && home.IsDirty) @@ -2127,14 +2316,18 @@ namespace GameCult.Caching IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source, - Func land) => Held(() => + Func, CultCommitOutcome> land) => Held(() => { if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); - var outcome = land(Validate(admitted, evicted, source)); + var home = Validate(admitted, evicted, source); + // The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch + // set, before land and before memory. land receives the admitted records with variants resolved. + var plan = Resolve(admitted, evicted, source); + var outcome = land(home, plan.Admitted); if (outcome != CultCommitOutcome.Committed) return outcome; - foreach (var change in Apply(admitted, evicted, source)) + foreach (var change in Apply(plan, evicted, source)) _held.Add((change, source != null)); return CultCommitOutcome.Committed; }); @@ -2262,12 +2455,215 @@ namespace GameCult.Caching } } - private List Apply( + private sealed class VariantPlan + { + public VariantPlan(IReadOnlyList admitted) + { + Admitted = admitted; + } + + // The admitted records, variants resolved. + public IReadOnlyList Admitted { get; } + + // Variants this admission did not write whose base chain it changed: re-resolved, same delta, same storedAt. + public List Dependents { get; } = new(); + + // The drift report of every variant resolved here; null when nothing was ignored. + public Dictionary Reports { get; } = new(StringComparer.Ordinal); + } + + // Resolves every admitted variant and every variant whose base chain this admission changes, base first, against + // the post-admission set. It refuses a broken variant naming the keys, and mutates nothing. + private VariantPlan Resolve( IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source) { - var changes = new List(admitted.Count + evicted.Count); + if (_variantKeys.Count == 0 && admitted.All(stored => stored.Variant == null)) + return new VariantPlan(admitted); + + var incoming = admitted.ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); + var gone = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); + CultStoredDocument? Lookup(string key) => + incoming.TryGetValue(key, out var found) ? found + : gone.Contains(key) ? null + : _entries.TryGetValue(key, out var held) ? held : null; + + // A variant is written only through the variant API: a plain record never lands on a variant's key, and the + // reverse is a rebase or a flatten, never an accident. A load reports what is on disk and is not a write. + if (source == null) + { + foreach (var stored in admitted) + { + _entries.TryGetValue(stored.Key.Value, out var present); + if (stored.Variant == null && !stored.Flattens && present?.Variant != null) + throw new InvalidOperationException( + $"Record {stored.Key.Value} is a variant of {present.Variant.BaseKey}; a plain write cannot replace it. " + + "Change it through the variant API, or Flatten it."); + if (stored.Variant != null && present != null && present.Variant == null) + throw new InvalidOperationException( + $"Record {stored.Key.Value} is a plain record; it cannot become a variant of {stored.Variant.BaseKey} in place. Use a new key."); + if (stored.Variant != null && present?.Variant != null && + present.Descriptor.DocumentType != stored.Descriptor.DocumentType) + throw new InvalidOperationException( + $"Variant {stored.Key.Value} is a {present.Descriptor.DocumentType.Name}; " + + $"rebasing it onto {stored.Variant.BaseKey} would make it a {stored.Descriptor.DocumentType.Name}."); + } + } + + var changed = new HashSet(incoming.Keys.Concat(gone), StringComparer.Ordinal); + var toResolve = new List(admitted.Where(stored => stored.Variant != null).Select(stored => stored.Key.Value)); + foreach (var variantKey in _variantKeys) + { + if (incoming.ContainsKey(variantKey) || gone.Contains(variantKey)) + continue; + var next = _entries[variantKey].Variant!.BaseKey; + for (var steps = 0; steps <= _entries.Count; steps++) + { + if (changed.Contains(next)) + { + toResolve.Add(variantKey); + break; + } + + var link = Lookup(next); + if (link?.Variant == null) + break; + next = link.Variant.BaseKey; + } + } + + if (toResolve.Count == 0) + return new VariantPlan(admitted); + + var pending = new HashSet(toResolve, StringComparer.Ordinal); + var done = new Dictionary(StringComparer.Ordinal); + var reports = new Dictionary(StringComparer.Ordinal); + var stack = new List(); + foreach (var key in toResolve) + ResolveOne(key); + + var plan = new VariantPlan(admitted.Select(stored => stored.Variant != null ? done[stored.Key.Value] : stored).ToList()); + foreach (var key in toResolve.Where(key => !incoming.ContainsKey(key))) + plan.Dependents.Add(done[key]); + foreach (var pair in reports) + plan.Reports[pair.Key] = pair.Value; + return plan; + + CultStoredDocument ResolveOne(string key) + { + if (done.TryGetValue(key, out var finished)) + return finished; + var variant = Lookup(key)!; + if (stack.Contains(key)) + throw CycleRefusal(stack.SkipWhile(entry => entry != key).Append(key)); + stack.Add(key); + + var delta = variant.Variant!; + var baseStored = Lookup(delta.BaseKey) ?? throw MissingBaseRefusal(key, delta.BaseKey, gone.Contains(delta.BaseKey)); + if (baseStored.Variant != null && pending.Contains(delta.BaseKey)) + baseStored = ResolveOne(delta.BaseKey); + + var descriptor = variant.Descriptor; + if (_codec == null) + throw new InvalidOperationException( + $"Variant {key} needs a codec to resolve; this cache has none. Create it with CultCacheMessagePack.Create."); + if (baseStored.Descriptor.DocumentType != descriptor.DocumentType) + throw new InvalidOperationException( + $"Variant {key} is a {descriptor.DocumentType.Name} but its base {delta.BaseKey} is a {baseStored.Descriptor.DocumentType.Name}; " + + "a variant has its base's concrete type."); + + var type = descriptor.DocumentType; + var kept = new List>(); + var ignored = new List(); + var warnings = new List(); + foreach (var entry in delta.Overrides) + { + if (entry.Op != CultOverrideOp.Set || entry.Path.Count != 1 || entry.Path[0].ElementId.Length != 0 || entry.Id.Length != 0) + throw new InvalidOperationException( + $"Variant {key} carries {entry.Op} at {entry.PathText}; this runtime resolves top-level Set overrides only."); + var slot = entry.Path[0].Slot; + var member = descriptor.RichMembers.FirstOrDefault(candidate => candidate.Slot == slot); + if (member == null) + { + var name = delta.SlotNames != null && delta.SlotNames.TryGetValue(slot, out var persistedName) + ? $"'{persistedName}'" : "unnamed"; + ignored.Add(slot); + warnings.Add(new CultSchemaMigrationWarning + { + Code = "ignored_extra_slot", + Message = $"Variant {key} overrides slot {slot} ({name}), which {type.Name} no longer has; the override is ignored and the variant inherits the base value." + }); + continue; + } + + try + { + _codec.Deserialize(member.MemberType, type, entry.Value); + } + catch (Exception exception) + { + ignored.Add(slot); + warnings.Add(new CultSchemaMigrationWarning + { + Code = "ignored_undecodable_override", + Message = $"Variant {key} overrides slot {slot} ('{member.Member.Name}') with a value that no longer decodes as {member.MemberType.Name} " + + $"({exception.GetBaseException().Message}); the override is ignored and the variant inherits the base value." + }); + continue; + } + + kept.Add(new KeyValuePair(slot, entry.Value)); + } + + var payload = _codec.Serialize(baseStored.Document, type, type); + var document = _codec.Deserialize(type, type, kept.Count == 0 ? payload : _codec.Overlay(payload, kept)) + ?? throw new InvalidOperationException($"Variant {key} resolved to nothing."); + + // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. + if (descriptor.NameAccessor?.Invoke(document) is { Length: > 0 } variantName) + { + for (var ancestor = baseStored; ancestor != null; + ancestor = ancestor.Variant == null ? null : done.TryGetValue(ancestor.Variant.BaseKey, out var above) ? above : Lookup(ancestor.Variant.BaseKey)) + { + if (descriptor.NameAccessor(ancestor.Document) == variantName) + throw new InvalidOperationException( + $"Variant {key} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); + } + } + + reports[key] = ignored.Count == 0 + ? null + : new CultSchemaMigrationReport + { + PersistedSchemaId = descriptor.SchemaId, + LocalSchemaId = descriptor.SchemaId, + PersistedSchemaName = descriptor.SchemaName, + LocalSchemaName = descriptor.SchemaName, + Kind = CultSchemaMigrationKind.CompatibleDrift, + IgnoredExtraSlots = ignored.OrderBy(slot => slot).ToArray(), + Warnings = warnings.ToArray() + }; + stack.RemoveAt(stack.Count - 1); + return done[key] = variant.Resolved(document); + } + } + + private static InvalidOperationException CycleRefusal(IEnumerable keys) => + new($"Variant bases form a cycle: {string.Join(" -> ", keys)}."); + + private static InvalidOperationException MissingBaseRefusal(string variant, string baseKey, bool removed) => + new(removed + ? $"Record {baseKey} cannot go: variant {variant} is based on it. Remove or rebase the variant in the same commit." + : $"Variant {variant} is based on {baseKey}, which does not exist."); + + private List Apply( + VariantPlan plan, + IReadOnlyList evicted, + CacheBackingStore? source) + { + var admitted = plan.Admitted; + var changes = new List(admitted.Count + evicted.Count + plan.Dependents.Count); foreach (var stored in evicted) { if (!_entries.TryGetValue(stored.Key.Value, out var existing)) @@ -2275,10 +2671,12 @@ namespace GameCult.Caching Unindex(existing); _entries.Remove(stored.Key.Value); _handles.Remove(existing.Document); + _variantKeys.Remove(stored.Key.Value); + _variantReports.Remove(stored.Key.Value); changes.Add(new Change(CultCacheDocumentChangeKind.Removed, existing, null, existing.Document, ++_sequence)); } - foreach (var stored in admitted) + foreach (var stored in admitted.Concat(plan.Dependents)) { _entries.TryGetValue(stored.Key.Value, out var previous); if (previous != null) @@ -2290,6 +2688,14 @@ namespace GameCult.Caching _entries[stored.Key.Value] = stored; _handles.AddOrUpdate(stored.Document, new KeyBox(stored.Key)); Index(stored); + if (stored.Variant == null) + _variantKeys.Remove(stored.Key.Value); + else + _variantKeys.Add(stored.Key.Value); + if (plan.Reports.TryGetValue(stored.Key.Value, out var report) && report != null) + _variantReports[stored.Key.Value] = report; + else + _variantReports.Remove(stored.Key.Value); changes.Add(new Change( previous == null ? CultCacheDocumentChangeKind.Added : CultCacheDocumentChangeKind.Updated, stored, @@ -2486,12 +2892,14 @@ namespace GameCult.Caching protected CultPersistedRecord ToPersistedRecord(CultStoredDocument entry, Func serializePayload) { + // A variant persists its delta and an empty payload; the resolved view is never written. return new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt, - Payload = serializePayload(entry.Document) + Payload = entry.Variant != null ? Array.Empty() : serializePayload(entry.Document), + Variant = entry.Variant == null ? null : new CultVariantDelta(entry.Variant.BaseKey, entry.Variant.Overrides) }; } @@ -2501,6 +2909,21 @@ namespace GameCult.Caching Func deserializePayload) { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); + if (record.Variant != null) + { + // The cache resolves it against its base when it admits the load; the catalog names the slots the type may have dropped. + var persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, record.SchemaId, StringComparison.Ordinal)); + return new CultStoredDocument( + new CultRecordKey(record.Key), + record.StoredAt, + resolution.Descriptor, + new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides) + { + SlotNames = persisted?.Members.ToDictionary(member => member.Slot, member => member.MemberName) + }, + null); + } + var document = deserializePayload(resolution.Descriptor.DocumentType, record.Payload); return new CultStoredDocument( new CultRecordKey(record.Key), @@ -2675,6 +3098,10 @@ namespace GameCult.Caching var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { + // v2 exactly when the store holds a variant: a store without one stays byte-identical v1. + FormatVersion = ordered.Any(record => record.Variant != null) + ? CultPersistedStoreSnapshot.FormatV2 + : CultPersistedStoreSnapshot.FormatV1, SchemaCatalog = catalog .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) .Select(group => group.First()) diff --git a/src/GameCult.Caching/CultInspectorModel.cs b/src/GameCult.Caching/CultInspectorModel.cs index e5b5581f..99d82205 100644 --- a/src/GameCult.Caching/CultInspectorModel.cs +++ b/src/GameCult.Caching/CultInspectorModel.cs @@ -292,22 +292,24 @@ namespace GameCult.Caching typeof(int), typeof(long), typeof(uint), typeof(short), typeof(ushort), typeof(byte), typeof(sbyte) }; - private readonly Func _serialize; - private readonly Func _deserialize; + private readonly CultCodec _codec; private readonly Dictionary _shapes = new Dictionary(); private readonly Dictionary _metadata = new Dictionary(); - // serialize (value, value type, owning document type) and deserialize are the store's codec - // (CultCacheMessagePack.CreateInspectorModel for .cc stores). Edits clone documents through both; dictionary keys - // compare by serialize under their owning document, exactly as the store writes them inside it, so serialize must - // take any value, not only documents. - public CultInspectorModel(CultDocumentRegistry registry, Func serialize, Func deserialize) + // The codec is the store's (CultCacheMessagePack.CreateInspectorModel for .cc stores), the one a CultCache resolves + // variants with. Edits clone documents through it; dictionary keys compare by its Serialize under their owning + // document, exactly as the store writes them inside it. + public CultInspectorModel(CultDocumentRegistry registry, CultCodec codec) { Registry = registry ?? throw new ArgumentNullException(nameof(registry)); - _serialize = serialize ?? throw new ArgumentNullException(nameof(serialize)); - _deserialize = deserialize ?? throw new ArgumentNullException(nameof(deserialize)); + _codec = codec ?? throw new ArgumentNullException(nameof(codec)); } + private byte[] _serialize(object value, Type type, Type document) => _codec.Serialize(value, type, document); + + private object _deserialize(Type type, byte[] bytes) => + _codec.Deserialize(type, type, bytes) ?? throw new InvalidOperationException($"The codec returned null for {type.FullName}."); + public CultDocumentRegistry Registry { get; } public static string RecordLabel(CultStoredDocument record) diff --git a/src/GameCult.Caching/CultVariants.cs b/src/GameCult.Caching/CultVariants.cs new file mode 100644 index 00000000..08882c79 --- /dev/null +++ b/src/GameCult.Caching/CultVariants.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace GameCult.Caching +{ + // The store's encoding, as the cache needs it: the one pair CultInspectorModel and variant resolution share. + // Serialize takes any value, not only documents, under its owning document type's options; Deserialize is its + // inverse (a document type is checked against the registry, a member type is not); Overlay replaces slots of a + // serialized document payload with already-encoded member values and touches nothing else. + public sealed class CultCodec + { + public CultCodec( + Func serialize, + Func deserialize, + Func>, byte[]> overlay) + { + Serialize = serialize ?? throw new ArgumentNullException(nameof(serialize)); + Deserialize = deserialize ?? throw new ArgumentNullException(nameof(deserialize)); + Overlay = overlay ?? throw new ArgumentNullException(nameof(overlay)); + } + + // (value, value type, owning document type) + public Func Serialize { get; } + + // (value type, owning document type, bytes) + public Func Deserialize { get; } + + // (document payload, slot and encoded value pairs) + public Func>, byte[]> Overlay { get; } + } + + public enum CultOverrideOp + { + Set = 0, + Insert = 1, + Remove = 2 + } + + // One step of a member path: the MessagePack slot, and the id of the object-list element the step enters ("" when it + // does not enter one). + public sealed class CultPathStep + { + public CultPathStep(int slot, string elementId = "") + { + Slot = slot; + ElementId = elementId ?? string.Empty; + } + + public int Slot { get; } + public string ElementId { get; } + + public override string ToString() => ElementId.Length == 0 ? $"[{Slot}]" : $"[{Slot}:{ElementId}]"; + } + + // The wire shape [op, path[], id, value]; value is the override's already-encoded MessagePack value. + public sealed class CultVariantOverride + { + public CultVariantOverride(CultOverrideOp op, IReadOnlyList path, string id, byte[] value) + { + Op = op; + Path = path ?? throw new ArgumentNullException(nameof(path)); + Id = id ?? string.Empty; + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + public CultOverrideOp Op { get; } + public IReadOnlyList Path { get; } + public string Id { get; } + public byte[] Value { get; } + + // A top-level Set: the whole member at slot takes the encoded value. + public static CultVariantOverride Set(int slot, byte[] value) => + new(CultOverrideOp.Set, new[] { new CultPathStep(slot) }, string.Empty, value); + + internal string PathText => string.Concat(Path.Select(step => step.ToString())); + } + + // What a variant record stores: its base and its overrides. Never the resolved document. + public sealed class CultVariantDelta + { + public CultVariantDelta(string baseKey, IReadOnlyList overrides) + { + BaseKey = baseKey ?? throw new ArgumentNullException(nameof(baseKey)); + Overrides = overrides ?? throw new ArgumentNullException(nameof(overrides)); + } + + public string BaseKey { get; } + public IReadOnlyList Overrides { get; } + + // Member names by slot from the catalog the delta was loaded under, for warnings about slots the type dropped. + // Not persisted. + internal IReadOnlyDictionary? SlotNames { get; set; } + } +} diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 46bfb47e..67cc4cdf 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -156,6 +156,8 @@ namespace GameCult.Mesh $"CultCache document '{path}' record '{key.Value}' has schema '{record.SchemaId}', expected '{expectedSchemaId}'."); } + RefuseVariant(path, record); + if (!PublishesSchema(snapshot.SchemaCatalog, expectedSchemaId)) { throw new InvalidDataException( @@ -189,6 +191,8 @@ namespace GameCult.Mesh $"CultCache document '{path}' has schema '{record.SchemaId}', expected '{expectedSchemaId}'."); } + RefuseVariant(path, record); + if (!PublishesSchema(snapshot.SchemaCatalog, expectedSchemaId)) { throw new InvalidDataException( @@ -198,6 +202,15 @@ namespace GameCult.Mesh return record.Payload; } + // A variant record's payload is empty; its document exists only as the cache resolves it against its base. + private static void RefuseVariant(string path, CultPersistedRecord record) + { + if (record.Variant != null) + throw new NotSupportedException( + $"CultCache document '{path}' record '{record.Key}' is a variant of '{record.Variant.BaseKey}'; " + + "CultMesh does not resolve document variants, open the store with a CultCache."); + } + private static void WriteSingleFileDocumentPayload( string path, CultRecordKey key, diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index be114002..60446d31 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -393,7 +393,7 @@ namespace GameCult.Networking stored = stored.Where(entry => rowFilter(entry.Descriptor, entry.Key)); var rows = stored .Select(entry => new CultNetSelectionEvaluator.Row( - entry.Descriptor, entry.Key, entry.Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) + entry.Descriptor, entry.Key, RefuseVariant(entry).Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); // R-Q: a page's asOf is exact only when every matched row is committed through one shard's @@ -487,7 +487,7 @@ namespace GameCult.Networking stored = stored.Where(entry => rowFilter(entry.Descriptor, entry.Key)); var rows = stored .Select(entry => new CultNetSelectionEvaluator.Row( - entry.Descriptor, entry.Key, entry.Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) + entry.Descriptor, entry.Key, RefuseVariant(entry).Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); @@ -549,6 +549,13 @@ namespace GameCult.Networking }; } + // CultNet carries no variant deltas yet, and a resolved view sent as a plain record would be a lie about the store. + private static CultStoredDocument RefuseVariant(CultStoredDocument entry) => + entry.Variant == null + ? entry + : throw new NotSupportedException( + $"Record '{entry.Key.Value}' is a variant of '{entry.Variant.BaseKey}'; CultNet does not carry document variants yet."); + /// /// Builds the wire record for one evaluated row. The shared path under snapshot and change /// (docs/cultnet-selection-cut.md, section 8's "shared paths"). From fc78b9ad2b6cad16766959e402f13d74e8bc02b6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:52:35 +0200 Subject: [PATCH 02/16] Test the C# variant path and add the C#-written variant store as a shared vector VariantTests drive the cache batch API and the real single-file store. The C1 vector was written by the code under test through that path; C# reads it and rewrites it byte for byte, and the other runtimes' tests read the same bytes next. The C0 tests that expected C# to refuse a v2 store now expect what C1 does with it. Co-Authored-By: Claude Sonnet 5 --- .../PreCut2StoreFormatTests.cs | 25 +- tests/GameCult.Caching.Tests/VariantTests.cs | 700 ++++++++++++++++++ .../CultMeshVariantRefusalTests.cs | 12 +- .../CultNetDocumentRegistryVariantTests.cs | 78 ++ .../document-variants-c1/.gitattributes | 3 + .../variant-store.msgpack | Bin 0 -> 1391 bytes 6 files changed, 812 insertions(+), 6 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/VariantTests.cs create mode 100644 tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs create mode 100644 tests/vectors/document-variants-c1/.gitattributes create mode 100644 tests/vectors/document-variants-c1/variant-store.msgpack diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index 479eb762..181eca56 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -131,11 +131,30 @@ namespace GameCult.Caching.Tests Assert.That(message, Does.Contain(ItemSchemaId)); } + // C1 reads v2. This vector was written by hand before C1 existed and its variant keeps its base's name, so a + // cache with a codec refuses it under R6, naming both records; a cache without one cannot resolve it at all. [Test] - public void SingleFileRefusesVariantStoreByVersionOrRecord() + public void SingleFileRefusesTheHandWrittenVariantVectorNamingTheRecords() { - Assert.That(Refusal("variant-v2.msgpack"), - Does.Contain("cultcache.store.v2").Or.Contain("item:anvil-big")); + var root = Path.Combine(Path.GetTempPath(), $"cultlib-vector-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + var file = Path.Combine(root, "store.msgpack"); + File.Copy(VectorPath("variant-v2.msgpack"), file); + + using var withCodec = new CultCache(CultDocumentRegistry.Shared, CultCacheMessagePack.CreateCodec(CultDocumentRegistry.Shared)); + var named = Assert.Throws(() => withCodec.AddBackingStore(new SingleFileMessagePackBackingStore(file)))!; + Assert.That(named.Message, Does.Contain("item:anvil-big").And.Contain("item:anvil")); + + using var without = new CultCache(); + var codecless = Assert.Throws(() => without.AddBackingStore(new SingleFileMessagePackBackingStore(file)))!; + Assert.That(codecless.Message, Does.Contain("item:anvil-big").And.Contain("codec")); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } } [Test] diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs new file mode 100644 index 00000000..501af538 --- /dev/null +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -0,0 +1,700 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // The contract is src/GameCult.Caching/Contracts/cultcache-persistence-format.md (variant records) and + // docs/document-variants-cut.md C1. Every input goes through the cache's own write API and the real single-file store. + public class VariantTests + { + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] + { + typeof(VariantGear), typeof(VariantOther), typeof(VariantGlobal) + }); + + private static readonly CultRecordKey BaseKey = new("laser"); + private static readonly CultRecordKey BigKey = new("laser-big"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-variants-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, bool readOnly = false, CultDocumentRegistry? registry = null) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry ?? Registry, ReadOnly = readOnly }); + + private static VariantGear Laser() => new() { Name = "laser", Power = 10, Code = "l1", Tags = { "beam" } }; + + private static void SeedBase(CultCache cache) => + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Laser(), BaseKey)); + + // laser-big: a variant of laser that overrides its name and power and inherits the rest. + private static void SeedBig(CultCache cache, CultRecordKey? key = null, CultRecordKey? baseKey = null, string name = "laser big", int power = 25) => + cache.Commit(batch => batch.UpsertVariant(key ?? BigKey, baseKey ?? BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), name), + cache.Override(nameof(VariantGear.Power), power) + })); + + private static void EditBase(CultCache cache, Action edit) => + cache.Commit(batch => + { + var edited = Laser(); + edit(edited); + batch.Upsert(typeof(VariantGear), edited, BaseKey); + }); + + private static InvalidOperationException Refused(TestDelegate write) => + Assert.Throws(write)!; + + // ---- resolution ---- + + [Test] + public void ALoadedVariantResolvesAgainstItsBase() + { + var path = PathOf("gear.cc"); + using (var writer = Open(path)) + { + SeedBase(writer); + SeedBig(writer); + } + + using var cache = Open(path); + var big = cache.Get(BigKey)!; + Assert.That(big.Name, Is.EqualTo("laser big")); + Assert.That(big.Power, Is.EqualTo(25)); + Assert.That(big.Code, Is.EqualTo("l1"), "an inherited member"); + Assert.That(big.Tags, Is.EqualTo(new[] { "beam" }), "an inherited list"); + Assert.That(cache.Get(BaseKey)!.Power, Is.EqualTo(10)); + Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Variant!.Overrides, Has.Count.EqualTo(2)); + } + + [Test] + public void AVariantOfAVariantResolvesBaseFirstAndFollowsTheRoot() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + var hugeKey = new CultRecordKey("laser-huge"); + SeedBig(cache, hugeKey, BigKey, name: "laser huge", power: 40); + + Assert.That(cache.Get(hugeKey)!.Tags, Is.EqualTo(new[] { "beam" })); + EditBase(cache, gear => gear.Tags.Add("pierce")); + + Assert.That(cache.Get(hugeKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + Assert.That(cache.Get(hugeKey)!.Power, Is.EqualTo(40)); + } + + [Test] + public void ABaseEditReachesItsVariantsThroughGetNameAndIndexAndOneWatchPublication() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("laser-coded"), BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser coded"), + cache.Override(nameof(VariantGear.Code), "l9") + })); + + var seen = new List<(string Key, long Sequence, string BigTagsAtThatMoment, string BigNameLookupTags)>(); + using var subscription = cache.Watch().Subscribe(change => seen.Add(( + change.Key.Value, + change.Sequence, + string.Join(",", cache.Get(BigKey)!.Tags), + string.Join(",", cache.GetByName("laser big")!.Tags)))); + + EditBase(cache, gear => { gear.Tags.Add("pierce"); gear.Code = "l2"; }); + + Assert.That(cache.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + Assert.That(cache.GetByName("laser big")!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + Assert.That(cache.GetByName("laser big")!.Power, Is.EqualTo(25), "the override survives the base edit"); + Assert.That(cache.GetByIndex("code", "l9")!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + // big inherited the old code; its stale index entry must be gone, not shadowing. + Assert.That(cache.GetByIndex("code", "l1"), Is.Null); + Assert.That(cache.GetByIndex("code", "l2")!.Name, Is.Not.Null); + + Assert.That(seen.Select(entry => entry.Key), Is.EquivalentTo(new[] { BaseKey.Value, BigKey.Value, "laser-coded" })); + Assert.That(seen.Select(entry => entry.Sequence), Is.EqualTo(Enumerable.Range((int)seen[0].Sequence, seen.Count).Select(n => (long)n)), + "one commit, one run of sequences with no gap"); + Assert.That(seen.Select(entry => entry.BigTagsAtThatMoment), Is.All.EqualTo("beam,pierce"), + "a subscriber observing the base change already reads the new variant"); + Assert.That(seen.Select(entry => entry.BigNameLookupTags), Is.All.EqualTo("beam,pierce")); + } + + [Test] + public void AReResolvedDependentKeepsItsStoredAtAndIsAnUpdatedChange() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + var before = cache.GetStored(BigKey)!; + var changes = new List>(); + using var subscription = cache.WatchRecord(BigKey).Subscribe(changes.Add); + + EditBase(cache, gear => gear.Tags.Add("pierce")); + + Assert.That(cache.GetStored(BigKey)!.StoredAt, Is.EqualTo(before.StoredAt), "a base change stamps no dependent"); + Assert.That(changes, Has.Count.EqualTo(1)); + Assert.That(changes[0].Kind, Is.EqualTo(CultCacheDocumentChangeKind.Updated)); + Assert.That(changes[0].PreviousDocument!.Tags, Is.EqualTo(new[] { "beam" })); + Assert.That(changes[0].Document!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + } + + [Test] + public void ExpectOnAVariantSurvivesABaseChangeButNotAChangeToItsDelta() + { + var path = PathOf("gear.cc"); + using (var seed = Open(path)) + { + SeedBase(seed); + SeedBig(seed); + } + + using var a = Open(path); + using var b = Open(path); + var seen = a.Get(BigKey)!; + + EditBase(b, gear => gear.Tags.Add("pierce")); + Assert.That(a.Commit(batch => + { + batch.Expect(BigKey, seen); + batch.Upsert(typeof(VariantOther), new VariantOther { Name = "marker" }, new CultRecordKey("marker")); + }), Is.True, "the base changed; the variant's stored delta did not"); + + a.PullAllBackingStoresAsync(); + seen = a.Get(BigKey)!; + SeedBig(b, power: 26); + Assert.That(a.Commit(batch => + { + batch.Expect(BigKey, seen); + batch.Upsert(typeof(VariantOther), new VariantOther { Name = "second" }, new CultRecordKey("second")); + }), Is.False, "another writer changed the variant's own delta"); + } + + // ---- refusals, each naming the keys ---- + + [Test] + public void ACycleOfBasesIsRefusedNamingEveryKey() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + var a = new CultRecordKey("cycle-a"); + var b = new CultRecordKey("cycle-b"); + + var staged = Refused(() => cache.Commit(batch => + { + batch.UpsertVariant(a, b); + batch.UpsertVariant(b, a); + })); + Assert.That(staged.Message, Does.Contain("cycle").And.Contain(a.Value).And.Contain(b.Value)); + + SeedBig(cache); + var huge = new CultRecordKey("laser-huge"); + SeedBig(cache, huge, BigKey, name: "laser huge"); + var rebased = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, huge))); + Assert.That(rebased.Message, Does.Contain("cycle").And.Contain(BigKey.Value).And.Contain(huge.Value)); + Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value), "nothing landed"); + } + + [Test] + public void AMissingBaseIsRefusedNamingBoth() + { + using var cache = Open(PathOf("gear.cc")); + var error = Refused(() => cache.UpsertVariantAsync(BigKey, new CultRecordKey("no-such-base")).GetAwaiter().GetResult()); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("no-such-base")); + Assert.That(cache.Get(BigKey), Is.Null); + } + + [Test] + public void ABaseInAnotherTypeIsRefusedOnRebaseAndOnLoad() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + SeedBase(cache); + SeedBig(cache); + cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, new CultRecordKey("other"))); + + var rebase = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other")))); + Assert.That(rebase.Message, Does.Contain(BigKey.Value).And.Contain("VariantOther")); + } + + // A store whose variant record names another type than its base's: the loader refuses it naming both keys. + var bad = PathOf("bad.cc"); + var gear = Registry.GetRequired(); + var other = Registry.GetRequired(); + File.WriteAllBytes(bad, CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + FormatVersion = CultPersistedStoreSnapshot.FormatV2, + SchemaCatalog = new[] { gear.ToCatalogEntry(), other.ToCatalogEntry() }, + Records = new[] + { + new CultPersistedRecord + { + Key = "gear-base", SchemaId = gear.SchemaId, StoredAt = "2026-01-01T00:00:00.0000000Z", + Payload = CultDocumentMessagePackSerialization.SerializeUntyped(Laser(), typeof(VariantGear), Registry) + }, + new CultPersistedRecord + { + Key = "other-variant", SchemaId = other.SchemaId, StoredAt = "2026-01-01T00:00:00.0000000Z", + Variant = new CultVariantDelta("gear-base", Array.Empty()) + } + } + })); + var load = Refused(() => Open(bad).Dispose()); + Assert.That(load.Message, Does.Contain("other-variant").And.Contain("gear-base")); + } + + [Test] + public void AVariantOfAGlobalIsRefused() + { + using var cache = Open(PathOf("gear.cc")); + var key = cache.UpsertAsync(new VariantGlobal { Value = "one" }).GetAwaiter().GetResult().Key; + var error = Refused(() => cache.UpsertVariantAsync(BigKey, key).GetAwaiter().GetResult()); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("global")); + } + + [Test] + public void RemovingABaseThatKeepsVariantsIsRefusedUnlessTheSameBatchRemovesOrRebasesThem() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + var spare = new CultRecordKey("spare"); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), new VariantGear { Name = "spare", Power = 3 }, spare)); + SeedBig(cache); + + var refused = Refused(() => cache.Commit(batch => batch.Remove(BaseKey))); + Assert.That(refused.Message, Does.Contain(BaseKey.Value).And.Contain(BigKey.Value)); + Assert.That(cache.Get(BaseKey), Is.Not.Null); + Assert.That(cache.Get(BigKey), Is.Not.Null); + + Assert.That(cache.Commit(batch => + { + batch.UpsertVariant(BigKey, spare, cache.GetStored(BigKey)!.Variant!.Overrides); + batch.Remove(BaseKey); + }), Is.True, "rebased in the same batch"); + Assert.That(cache.Get(BigKey)!.Code, Is.EqualTo(string.Empty), "now resolved from spare"); + + Assert.That(cache.Commit(batch => + { + batch.Remove(spare); + batch.Remove(BigKey); + }), Is.True, "removed in the same batch"); + } + + [Test] + public void AVariantThatKeepsItsBasesNameIsRefused() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + var error = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey).GetAwaiter().GetResult()); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("laser")); + + var sameName = Refused(() => SeedBig(cache, name: "laser")); + Assert.That(sameName.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value)); + + SeedBig(cache); + var grand = new CultRecordKey("laser-grand"); + var grandOfBase = Refused(() => SeedBig(cache, grand, BigKey, name: "laser")); + Assert.That(grandOfBase.Message, Does.Contain(grand.Value).And.Contain(BaseKey.Value), "the whole chain, not only the parent"); + } + + [Test] + public void ABaseEditThatWouldGiveAVariantItsBasesNameIsRefused() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + + var error = Refused(() => EditBase(cache, gear => gear.Name = "laser big")); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value)); + Assert.That(cache.Get(BaseKey)!.Name, Is.EqualTo("laser")); + } + + [Test] + public void APlainWriteAtAVariantsKeyIsRefusedByEveryPathAndFlattenIsTheExplicitWay() + { + var path = PathOf("gear.cc"); + using var cache = Open(path); + SeedBase(cache); + SeedBig(cache); + var plain = new VariantGear { Name = "laser big", Power = 99 }; + + Assert.That(Refused(() => cache.UpsertAsync(typeof(VariantGear), plain, BigKey).GetAwaiter().GetResult()).Message, + Does.Contain(BigKey.Value)); + Assert.That(Refused(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), plain, BigKey))).Message, + Does.Contain(BigKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Variant, Is.Not.Null); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25)); + + cache.FlattenAsync(BigKey).GetAwaiter().GetResult(); + var flat = cache.GetStored(BigKey)!; + Assert.That(flat.Variant, Is.Null); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25), "flatten keeps the resolved values"); + EditBase(cache, gear => gear.Tags.Add("pierce")); + Assert.That(cache.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam" }), "a flattened record no longer follows its base"); + cache.UpsertAsync(typeof(VariantGear), plain, BigKey).GetAwaiter().GetResult(); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(99), "and is an ordinary record again"); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "no variant left in the store"); + } + + [Test] + public void FlattenRefusesARecordThatIsNotAVariantAndAPlainRecordCannotBecomeOne() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + Assert.That(Refused(() => cache.FlattenAsync(BaseKey).GetAwaiter().GetResult()).Message, Does.Contain(BaseKey.Value)); + var spare = new CultRecordKey("spare"); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), new VariantGear { Name = "spare" }, spare)); + Assert.That(Refused(() => cache.UpsertVariantAsync(spare, BaseKey).GetAwaiter().GetResult()).Message, Does.Contain(spare.Value)); + } + + [Test] + public void ACacheWithoutACodecHoldsNoVariants() + { + using var cache = new CultCache(Registry); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Laser(), BaseKey)); + var error = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey).GetAwaiter().GetResult()); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("codec")); + } + + [Test] + public void OnlyTopLevelSetOverridesResolveInThisCut() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + var element = new CultVariantOverride( + CultOverrideOp.Set, new[] { new CultPathStep(3, "el-1"), new CultPathStep(0) }, string.Empty, new byte[] { 0x01 }); + var error = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey, new[] { element }).GetAwaiter().GetResult()); + Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("top-level")); + + var insert = new CultVariantOverride(CultOverrideOp.Insert, new[] { new CultPathStep(3) }, "new", new byte[] { 0xa1, 0x78 }); + Assert.That(Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey, new[] { insert }).GetAwaiter().GetResult()).Message, + Does.Contain(BigKey.Value).And.Contain("top-level")); + } + + // ---- soft drift: an override drifts exactly as the same member would on a plain record ---- + + [Test] + public void AnOverrideOfASlotTheTypeNoLongerHasIsIgnoredWithAWarningNamingVariantAndMember() + { + var path = PathOf("drift.cc"); + var wide = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }); + using (var writer = Open(path, registry: wide)) + { + writer.Commit(batch => + { + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); + batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] + { + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Power), 7), + writer.Override(nameof(DriftWide.Wing), "variant wing") + }); + }); + } + + var narrow = CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }); + using var cache = Open(path, registry: narrow); + var variant = cache.Get(new CultRecordKey("drift-variant"))!; + Assert.That(variant.Power, Is.EqualTo(7), "the override the type still has applies"); + Assert.That(variant.Name, Is.EqualTo("drift variant")); + + var report = cache.VariantMigrationReports.Single(); + Assert.That(report.Kind, Is.EqualTo(CultSchemaMigrationKind.CompatibleDrift)); + Assert.That(report.IgnoredExtraSlots, Is.EqualTo(new[] { 2 })); + var warning = report.Warnings.Single(); + Assert.That(warning.Code, Is.EqualTo("ignored_extra_slot"), "the code a dropped member gets on a plain record"); + Assert.That(warning.Message, Does.Contain("drift-variant").And.Contain("Wing")); + } + + [Test] + public void AnOverrideThatNoLongerDecodesIsIgnoredWithAWarningAndTheBaseValueApplies() + { + var path = PathOf("gear.cc"); + const int notAnInt = 1; // VariantGear.Power, [Key(1)] + using (var cache = Open(path)) + { + SeedBase(cache); + cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big"), + CultVariantOverride.Set(notAnInt, new byte[] { 0xa3, (byte)'a', (byte)'b', (byte)'c' }) + })); + var warning = cache.VariantMigrationReports.Single().Warnings.Single(); + Assert.That(warning.Message, Does.Contain(BigKey.Value).And.Contain("Power")); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(10), "inherits the base value"); + Assert.That(cache.Get(BigKey)!.Name, Is.EqualTo("laser big")); + } + + using var reloaded = Open(path); + Assert.That(reloaded.Get(BigKey)!.Power, Is.EqualTo(10)); + Assert.That(reloaded.VariantMigrationReports.Single().Warnings.Single().Message, Does.Contain("Power")); + } + + // ---- persisted format ---- + + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + [Test] + public void TheHeaderIsV1WithoutVariantsAndV2WithThem() + { + var path = PathOf("gear.cc"); + using var cache = Open(path); + SeedBase(cache); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + + SeedBig(cache); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2")); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var variant = snapshot.Records.Single(record => record.Key == BigKey.Value); + Assert.That(variant.Payload, Is.Empty, "the payload is the tripwire old readers refuse on"); + Assert.That(variant.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); + Assert.That(snapshot.Records.Single(record => record.Key == BaseKey.Value).Variant, Is.Null); + + cache.Commit(batch => batch.Remove(BigKey)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the last variant gone, the store is v1 again"); + } + + [Test] + public async Task AStoreWithoutVariantsRewritesByteIdenticalToTheOneWrittenBeforeVariantsExisted() + { + var fixture = Path.Combine(TestContext.CurrentContext.TestDirectory, "Fixtures", "pre-cut2", "single-file-v1", "store.msgpack"); + var copy = PathOf("store.msgpack"); + File.Copy(fixture, copy); + var before = File.ReadAllBytes(copy); + + using (var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(PreCut2FixtureItem), typeof(PreCut2FixtureNote) }))) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(copy)); + await cache.PullAllBackingStoresAsync(); + cache.BackingStores[0].PushAll(); + } + + Assert.That(File.ReadAllBytes(copy), Is.EqualTo(before)); + } + + [Test] + public void AVariantStoreRoundTripsByteIdenticalThroughAFlush() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + SeedBase(cache); + SeedBig(cache); + } + + var written = File.ReadAllBytes(path); + using (var cache = Open(path)) + { + cache.BackingStores[0].PushAll(); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(written)); + } + + [Test] + public void AReadOnlyStoreResolvesItsVariants() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + SeedBase(cache); + SeedBig(cache); + } + + var before = File.ReadAllBytes(path); + using var readOnly = Open(path, readOnly: true); + Assert.That(readOnly.Get(BigKey)!.Power, Is.EqualTo(25)); + Assert.That(readOnly.Get(BigKey)!.Code, Is.EqualTo("l1")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + } + + [Test] + public void ABaseEditFromAnotherWriterReachesVariantsWhenTheStoreIsPulled() + { + var path = PathOf("gear.cc"); + using var a = Open(path); + SeedBase(a); + SeedBig(a); + using var b = Open(path); + Assert.That(b.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam" })); + + EditBase(a, gear => gear.Tags.Add("pierce")); + b.PullAllBackingStoresAsync(); + + Assert.That(b.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" }), + "the pull carried only the base; the variant was re-resolved in the admitting hold"); + } + + [Test] + public void ARebaseResolvesFromTheNewBase() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + var spare = new CultRecordKey("spare"); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), new VariantGear { Name = "spare", Code = "s1", Tags = { "arc" } }, spare)); + + cache.Commit(batch => batch.UpsertVariant(BigKey, spare, cache.GetStored(BigKey)!.Variant!.Overrides)); + + Assert.That(cache.Get(BigKey)!.Tags, Is.EqualTo(new[] { "arc" })); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25)); + Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(spare.Value)); + } + + // ---- the other stores refuse loudly until their own cuts ---- + + [Test] + public void TheDirectoryStoreRefusesAVariantByPushAndByCommit() + { + using var cache = CultCacheMessagePack.Create(PathOf("dir.cc"), new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }); + SeedBase(cache); + + var push = Assert.Throws(() => cache.UpsertVariantAsync( + BigKey, BaseKey, new[] { cache.Override(nameof(VariantGear.Name), "laser big") }).GetAwaiter().GetResult())!; + Assert.That(push.Message, Does.Contain(BigKey.Value)); + var commit = Assert.Throws(() => SeedBig(cache))!; + Assert.That(commit.Message, Does.Contain(BigKey.Value)); + Assert.That(cache.Get(BigKey), Is.Null); + } + + // ---- shared vector ---- + + // Written once by the code under test through the production path; every runtime's tests read the committed bytes. + // CULTLIB_WRITE_VARIANT_VECTOR=1 prints the store as base64 instead of asserting. + [Test] + public void TheCommittedC1VectorResolvesAndRewritesByteIdentical() + { + var vector = VectorPath("variant-store.msgpack"); + var path = PathOf("gear.cc"); + if (Environment.GetEnvironmentVariable("CULTLIB_WRITE_VARIANT_VECTOR") == "1") + { + using (var cache = Open(path)) + { + SeedBase(cache); + SeedBig(cache); + } + + Console.WriteLine("VARIANT-VECTOR-BASE64:" + Convert.ToBase64String(File.ReadAllBytes(path))); + Assert.Pass("vector printed"); + } + + File.Copy(vector, path); + using (var cache = Open(path)) + { + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25)); + Assert.That(cache.Get(BigKey)!.Code, Is.EqualTo("l1")); + cache.BackingStores[0].PushAll(); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(File.ReadAllBytes(vector))); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2")); + } + + private static string VectorPath(string name) + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c1", name); + if (File.Exists(candidate) || Environment.GetEnvironmentVariable("CULTLIB_WRITE_VARIANT_VECTOR") == "1" && Directory.Exists(Path.GetDirectoryName(candidate))) + return candidate; + } + + if (Environment.GetEnvironmentVariable("CULTLIB_WRITE_VARIANT_VECTOR") == "1") + return name; + throw new FileNotFoundException($"Shared vector {name} not found above {TestContext.CurrentContext.TestDirectory}."); + } + + [CultDocument("tests.variant_gear", "tests.variant_gear.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class VariantGear + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + + [Key(2)] + [CultIndex("code")] + public string Code = string.Empty; + + [Key(3)] + public List Tags = new(); + } + + [CultDocument("tests.variant_other", "tests.variant_other.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class VariantOther + { + [Key(0)] + [CultName] + public string Name = string.Empty; + } + + [CultDocument("tests.variant_global", "tests.variant_global.v1")] + [MessagePackObject(AllowPrivate = true)] + [CultGlobal] + internal sealed class VariantGlobal + { + [Key(0)] + public string Value = string.Empty; + } + + // The same schema at two versions of the type: Wide has a member Narrow later dropped. + [CultDocument("tests.variant_drift", "tests.variant_drift.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class DriftWide + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + + [Key(2)] + public string Wing = string.Empty; + } + + [CultDocument("tests.variant_drift", "tests.variant_drift.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class DriftNarrow + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + } + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs index a5be069a..2521aa8a 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs @@ -45,10 +45,16 @@ public sealed class CultMeshVariantRefusalTests message.Should().Contain("item:anvil").And.Contain(ItemSchemaId); } + // Q6: CultMesh does not resolve variants. It refuses a variant key naming it and its base, and still reads the plain + // records of the same v2 store. [Test] - public void ReadSingleFileDocument_RefusesVariantStoreByVersionOrRecord() + public void ReadSingleFileDocument_RefusesAVariantKeyByNameButReadsItsPlainNeighbours() { - var message = Refusal("variant-v2.msgpack").Message; - (message.Contains("cultcache.store.v2") || message.Contains("item:anvil-big")).Should().BeTrue(message); + var refusal = Assert.Throws(() => + CultMesh.ReadSingleFileDocumentPayload(VectorPath("variant-v2.msgpack"), new CultRecordKey("item:anvil-big"), ItemSchemaId))!; + refusal.Message.Should().Contain("item:anvil-big").And.Contain("item:anvil'"); + + CultMesh.ReadSingleFileDocumentPayload(VectorPath("variant-v2.msgpack"), new CultRecordKey("item:bellows"), ItemSchemaId) + .Should().NotBeEmpty(); } } diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs new file mode 100644 index 00000000..46f91438 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -0,0 +1,78 @@ +#nullable enable +using System; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // Q6 (docs/document-variants-cut.md C1): CultNet carries no variant deltas yet, so building a row for one is refused + // loudly, on the v1 selection path and on the v0 snapshot path alike. + public sealed class CultNetDocumentRegistryVariantTests + { + private static readonly CultRecordKey BaseKey = new("variant-fixture-base"); + private static readonly CultRecordKey VariantKey = new("variant-fixture-big"); + + private static async Task<(CultCache Cache, CultNetDocumentRegistry Documents)> BuildFixtureAsync(bool withVariant) + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(BaseKey)); + if (withVariant) + await cache.UpsertVariantAsync(VariantKey, BaseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big"), + cache.Override(nameof(VariantWireFixture.Power), 9) + }); + return (cache, documents); + } + + [Test] + public async Task SelectionResponseRefusesAVariantRowNamingIt() + { + var (cache, documents) = await BuildFixtureAsync(withVariant: true); + var selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document }; + + var error = Assert.Throws(() => + documents.CreateSelectionResponse(cache, "m1", selection, ordinalOf: (_, _) => 1, asOf: 1))!; + + Assert.That(error.Message, Does.Contain(VariantKey.Value).And.Contain(BaseKey.Value)); + } + + [Test] + public async Task RawSnapshotResponseRefusesAVariantRowNamingIt() + { + var (cache, documents) = await BuildFixtureAsync(withVariant: true); + + var error = Assert.Throws(() => documents.CreateRawSnapshotResponse(cache, "m1"))!; + + Assert.That(error.Message, Does.Contain(VariantKey.Value)); + } + + [Test] + public async Task ACacheWithoutVariantsStillAnswersBothPaths() + { + var (cache, documents) = await BuildFixtureAsync(withVariant: false); + var selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document }; + + Assert.That(documents.CreateSelectionResponse(cache, "m1", selection, ordinalOf: (_, _) => 1, asOf: 1).Documents, Has.Length.EqualTo(1)); + Assert.That(documents.CreateRawSnapshotResponse(cache, "m2").Documents, Has.Length.EqualTo(1)); + } + + [CultDocument("variants.wire-fixture", "variants.wire-fixture.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class VariantWireFixture + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + } + } +} diff --git a/tests/vectors/document-variants-c1/.gitattributes b/tests/vectors/document-variants-c1/.gitattributes new file mode 100644 index 00000000..8efeaf12 --- /dev/null +++ b/tests/vectors/document-variants-c1/.gitattributes @@ -0,0 +1,3 @@ +# Refusal vectors are byte-exact; never convert line endings. +* binary +*.py text diff --git a/tests/vectors/document-variants-c1/variant-store.msgpack b/tests/vectors/document-variants-c1/variant-store.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..3604a68da1979eac264453f272960bea7d7ed467 GIT binary patch literal 1391 zcmcJOO^ee&7{|No&6`JWnyYP?nPieo5JC2^2qIz^FCsFTJkt#%sbr?rvh0Uwt=MCg z?m^JYg7;GW67|zKNn^CugW&c!&)fg^yv&(q#!8|`;EI__5X(YYx zhwslUW!lO3Bc6d3TDy8yyN-AYg-ZJQUxN&kiOV31Opp~mGgZ^RwnEgl@=f8Ye%sQc zm!(X1q%>r9R~Jd%Gnt8c0ulmg^}DQd5~*|><6T3=NAcSNRIk3?0{VDNG!3@Na~^d!QwP5 zC%mV2SdRK>3PPKuEBhF7C=y{mOq9O6wZ41%9NY8radYke`@g2M)}p+P&s+Gb)$;3d zc5qSM#i!a;orCeLmb1e}xu>0+)uJ5j?AoH7+}f`5{&8Ye<;nCSu-IVAmVip z>kxK=aOnCWVm89&^ZV=$LLoxmZq%aJKcslPh=6BrKUNf-DCzB)1XI_iKCMYwlTG`R VX=^-b&F348TLz;v9L(o8e*nV`+>rnP literal 0 HcmV?d00001 From 33664434f95b59253d0de50d32d9ef770dace13d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 04:21:58 +0200 Subject: [PATCH 03/16] Drop the unreachable single-record CultMesh variant refusal; tolerate duplicate catalog slots A variant needs its base beside it, so a single-record store can never hold one: that overload's refusal could not fire. SlotNames no longer throws on a catalog that repeats a slot. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 2 +- src/GameCult.Mesh/CultMeshSingleFileDocuments.cs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 24c20cfc..00b849a0 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2919,7 +2919,7 @@ namespace GameCult.Caching resolution.Descriptor, new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides) { - SlotNames = persisted?.Members.ToDictionary(member => member.Slot, member => member.MemberName) + SlotNames = persisted?.Members.GroupBy(member => member.Slot).ToDictionary(group => group.Key, group => group.First().MemberName) }, null); } diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 67cc4cdf..700b7a50 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -191,8 +191,6 @@ namespace GameCult.Mesh $"CultCache document '{path}' has schema '{record.SchemaId}', expected '{expectedSchemaId}'."); } - RefuseVariant(path, record); - if (!PublishesSchema(snapshot.SchemaCatalog, expectedSchemaId)) { throw new InvalidDataException( From b93934f6b432bff62efc144cc6b6308d9df4ee6a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 04:21:59 +0200 Subject: [PATCH 04/16] Pin the variant rules a mutant survived or that lost their only test Reader refusals (v1 header over a variant, payload on a variant, unknown op), a load that finds a variant flattened by another writer, a re-resolution reaching a read-only cache, a base removed in the same batch as its new variant, and plain-record-to-variant refused without the name rule masking it. Co-Authored-By: Claude Sonnet 5 --- tests/GameCult.Caching.Tests/VariantTests.cs | 99 +++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 501af538..fec33c96 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -304,6 +304,14 @@ namespace GameCult.Caching.Tests batch.Remove(spare); batch.Remove(BigKey); }), Is.True, "removed in the same batch"); + + SeedBase(cache); + var removedInTheBatch = Refused(() => cache.Commit(batch => + { + batch.Remove(BaseKey); + batch.UpsertVariant(BigKey, BaseKey); + })); + Assert.That(removedInTheBatch.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value)); } [Test] @@ -371,7 +379,9 @@ namespace GameCult.Caching.Tests Assert.That(Refused(() => cache.FlattenAsync(BaseKey).GetAwaiter().GetResult()).Message, Does.Contain(BaseKey.Value)); var spare = new CultRecordKey("spare"); cache.Commit(batch => batch.Upsert(typeof(VariantGear), new VariantGear { Name = "spare" }, spare)); - Assert.That(Refused(() => cache.UpsertVariantAsync(spare, BaseKey).GetAwaiter().GetResult()).Message, Does.Contain(spare.Value)); + var named = new[] { cache.Override(nameof(VariantGear.Name), "spare variant") }; + Assert.That(Refused(() => cache.UpsertVariantAsync(spare, BaseKey, named).GetAwaiter().GetResult()).Message, Does.Contain(spare.Value)); + Assert.That(cache.GetStored(spare)!.Variant, Is.Null, "the plain record is still plain"); } [Test] @@ -529,10 +539,16 @@ namespace GameCult.Caching.Tests SeedBig(cache); } - var before = File.ReadAllBytes(path); using var readOnly = Open(path, readOnly: true); Assert.That(readOnly.Get(BigKey)!.Power, Is.EqualTo(25)); Assert.That(readOnly.Get(BigKey)!.Code, Is.EqualTo("l1")); + + // A base edit another writer commits reaches the read-only cache's variant when it pulls: a re-resolution is no write. + using var writer = Open(path); + EditBase(writer, gear => gear.Tags.Add("pierce")); + var before = File.ReadAllBytes(path); + readOnly.PullAllBackingStoresAsync(); + Assert.That(readOnly.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); } @@ -553,6 +569,24 @@ namespace GameCult.Caching.Tests "the pull carried only the base; the variant was re-resolved in the admitting hold"); } + [Test] + public void AFlattenFromAnotherWriterReachesACacheHoldingTheVariant() + { + var path = PathOf("gear.cc"); + using var a = Open(path); + SeedBase(a); + SeedBig(a); + using var b = Open(path); + Assert.That(b.GetStored(BigKey)!.Variant, Is.Not.Null); + + a.FlattenAsync(BigKey).GetAwaiter().GetResult(); + a.FlushAsync().GetAwaiter().GetResult(); + b.PullAllBackingStoresAsync(); + + Assert.That(b.GetStored(BigKey)!.Variant, Is.Null, "a load reports what is on disk; it is not a plain write refused at a variant key"); + Assert.That(b.Get(BigKey)!.Power, Is.EqualTo(25)); + } + [Test] public void ARebaseResolvesFromTheNewBase() { @@ -569,6 +603,67 @@ namespace GameCult.Caching.Tests Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(spare.Value)); } + // ---- readers ---- + + private string WriteStore(params CultPersistedRecord[] records) => WriteStore(CultPersistedStoreSnapshot.FormatV2, records); + + private string WriteStore(string formatVersion, params CultPersistedRecord[] records) + { + var path = PathOf($"hand-{Guid.NewGuid():N}.cc"); + var gear = Registry.GetRequired(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + FormatVersion = formatVersion, + SchemaCatalog = new[] { gear.ToCatalogEntry() }, + Records = records + })); + return path; + } + + private static CultPersistedRecord PlainGear(string key) => new() + { + Key = key, + SchemaId = Registry.GetRequired().SchemaId, + StoredAt = "2026-01-01T00:00:00.0000000Z", + Payload = CultDocumentMessagePackSerialization.SerializeUntyped(Laser(), typeof(VariantGear), Registry) + }; + + private static CultPersistedRecord VariantGearRecord(string key, string baseKey, byte[]? payload = null, CultOverrideOp op = CultOverrideOp.Set) => new() + { + Key = key, + SchemaId = Registry.GetRequired().SchemaId, + StoredAt = "2026-01-01T00:00:00.0000000Z", + Payload = payload ?? Array.Empty(), + Variant = new CultVariantDelta(baseKey, new[] + { + new CultVariantOverride(op, new[] { new CultPathStep(0) }, string.Empty, new byte[] { 0xa1, (byte)'x' }) + }) + }; + + [Test] + public void AVariantRecordUnderAV1HeaderIsRefusedNamingTheRecord() + { + var path = WriteStore(CultPersistedStoreSnapshot.FormatV1, PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base")); + var error = Assert.Throws(() => Open(path).Dispose())!; + Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("cultcache.store.v2")); + } + + [Test] + public void AVariantRecordWithAPayloadIsRefusedNamingTheRecord() + { + var path = WriteStore(PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base", payload: new byte[] { 0x90 })); + var error = Assert.Throws(() => Open(path).Dispose())!; + Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("payload")); + } + + [Test] + public void AnOverrideWithAnUnknownOpIsRefusedNamingTheRecord() + { + var path = WriteStore(PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base", op: (CultOverrideOp)7)); + var error = Assert.Throws(() => Open(path).Dispose())!; + Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("op 7")); + } + // ---- the other stores refuse loudly until their own cuts ---- [Test] From 136adce9ff02145f248f2f4cb1ebd91f90db6787 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 04:21:59 +0200 Subject: [PATCH 05/16] Prove TypeScript, Python and Rust refuse the C#-written variant store by name Co-Authored-By: Claude Sonnet 5 --- packages/cultcache-py/tests/test_cultcache.py | 6 ++++++ packages/cultcache-rs/src/lib.rs | 10 ++++++++++ packages/cultcache-ts/test/cult-cache.test.ts | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index b4168690..6001c905 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -829,6 +829,12 @@ class CultCacheTests(unittest.TestCase): message = str(caught.exception) self.assertTrue("cultcache.store.v2" in message or "item:anvil-big" in message, message) + def test_single_file_refuses_the_csharp_written_variant_store_by_version_or_record(self) -> None: + with self.assertRaises(ValueError) as caught: + self._pull_vector("../document-variants-c1/variant-store.msgpack") + message = str(caught.exception) + self.assertTrue("cultcache.store.v2" in message or "laser-big" in message, message) + def test_v1_store_written_at_the_base_commit_still_reads_byte_for_byte(self) -> None: envelopes = self._pull_vector("v1-base.msgpack") self.assertEqual([(e.key, e.type) for e in envelopes], [("alpha", "vectors.item"), ("beta", "vectors.item")]) diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index e48ebad8..c9a7d282 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -4679,6 +4679,16 @@ mod tests { ); } + // The store the C# runtime wrote through its own variant API (tests/vectors/document-variants-c1). + #[test] + fn single_file_refuses_the_csharp_written_variant_store_by_version_or_record() { + let message = refusal("../document-variants-c1/variant-store.msgpack"); + assert!( + message.contains("cultcache.store.v2") || message.contains("laser-big"), + "{message}" + ); + } + #[test] fn v1_store_written_at_the_base_commit_still_reads_byte_for_byte() -> Result<()> { let envelopes = pull_vector("v1-base.msgpack")?; diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index a743ba00..8753af47 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1443,6 +1443,13 @@ test("SingleFileMessagePackBackingStore refuses a variant store by version or re ); }); +test("SingleFileMessagePackBackingStore refuses the C#-written variant store by version or record", async () => { + await assert.rejects( + () => pullVector("../document-variants-c1/variant-store.msgpack"), + (error: Error) => error.message.includes("cultcache.store.v2") || error.message.includes("laser-big"), + ); +}); + test("CultCache inspector refuses the same vectors by name", async () => { const inspect = async (name: string) => inspectCultCacheBytes(name, await readFile(join(variantVectors, name))); await assert.rejects(() => inspect("unknown-header.msgpack"), /cultcache\.store\.v9/u); From cdb8ee5dcc82518719518070b161ff955967b404 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 06:58:51 +0200 Subject: [PATCH 06/16] CultNet refuses a variant on both live-change paths, loudly, by key A load or write that landed a variant reached CultNet subscribers as a plain "added" record carrying the full resolved payload; a later base edit published only the base, so the subscriber's copy of the variant went stale. Under Q6 CultNet carries no variants until its own cut. Both live paths now refuse a change whose key holds a variant, after selection and authorization match, and send the matching subscriber a CultNetErrorMessage naming the variant and its base. Refused loudly rather than omitted: an omission is a silent hole in the subscriber's view, and the snapshot path already refuses by throwing. Tests: a real load through a real RUDP subscription never delivers the variant as a record and delivers the typed error; the database server's change builder refuses the same change. Co-Authored-By: Claude Sonnet 5 --- .../CultNetDatabaseServer.cs | 16 ++- .../CultNetDatabaseSubscriptionServer.cs | 14 +- .../CultNetDocumentRegistry.cs | 15 +- .../CultNetDocumentRegistryVariantTests.cs | 129 ++++++++++++++++++ 4 files changed, 167 insertions(+), 7 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 1a66dc2a..0dc489ed 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -504,7 +504,18 @@ namespace GameCult.Networking { return _database.WatchAllChanges().Subscribe(change => { - var outbound = CreateChangeMessage(change, subscriptionId, selection); + CultNetDatabaseChangeRawMessage? outbound; + try + { + outbound = CreateChangeMessage(change, subscriptionId, selection); + } + catch (NotSupportedException refusal) + { + // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. + peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + return; + } + if (outbound != null) { peer.SendCultNet(outbound); @@ -549,11 +560,12 @@ namespace GameCult.Networking }; } + CultNetDocumentRegistry.RefuseVariantChange(_database.Cache, key); return new CultNetDatabaseChangeRawMessage { MessageId = Guid.NewGuid().ToString("N"), SubscriptionId = subscriptionId, - ChangeKind = kind == CultNetDatabaseChangeKind.Added ? "added" : "updated", + ChangeKind = kind ==CultNetDatabaseChangeKind.Added ? "added" : "updated", Document = _database.Documents.ToRawRecord(descriptor, key, document, DateTimeOffset.UtcNow.ToString("O")) }; } diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index 2ccb031c..359a9dfb 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -315,7 +315,18 @@ namespace GameCult.Networking } var sourceRecordKey = ResolveChangeRecordKey(change); - var matched = CreateMatchedRecord(change, request, peer); + CultNetRawDocumentRecord? matched; + try + { + matched = CreateMatchedRecord(change, request, peer); + } + catch (NotSupportedException refusal) + { + // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. + peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + return; + } + ApplyProjectedChange( projection, sourceRecordKey, @@ -541,6 +552,7 @@ namespace GameCult.Networking if (kind == CultNetDatabaseChangeKind.Removed || document == null) return null; + CultNetDocumentRegistry.RefuseVariantChange(_database.Cache, key); return _database.Documents.ToRawRecord(descriptor, key, document, DateTimeOffset.UtcNow.ToString("O")); } diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 60446d31..356761f5 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -551,10 +551,17 @@ namespace GameCult.Networking // CultNet carries no variant deltas yet, and a resolved view sent as a plain record would be a lie about the store. private static CultStoredDocument RefuseVariant(CultStoredDocument entry) => - entry.Variant == null - ? entry - : throw new NotSupportedException( - $"Record '{entry.Key.Value}' is a variant of '{entry.Variant.BaseKey}'; CultNet does not carry document variants yet."); + entry.Variant == null ? entry : throw VariantRefusal(entry); + + // The live-change funnel's refusal: a load or write that lands a variant must not reach a subscriber as a record. + internal static void RefuseVariantChange(CultCache cache, CultRecordKey key) + { + if (cache.GetStored(key) is { Variant: not null } entry) + throw VariantRefusal(entry); + } + + private static NotSupportedException VariantRefusal(CultStoredDocument entry) => + new($"Record '{entry.Key.Value}' is a variant of '{entry.Variant!.BaseKey}'; CultNet does not carry document variants yet."); /// /// Builds the wire record for one evaluated row. The shared path under snapshot and change diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 46f91438..2a8ddc58 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -1,5 +1,12 @@ #nullable enable using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Reflection; +using System.Threading; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; @@ -63,6 +70,128 @@ namespace GameCult.Networking.Tests Assert.That(documents.CreateRawSnapshotResponse(cache, "m2").Documents, Has.Length.EqualTo(1)); } + // The live-change funnel: a load that lands a variant reaches a subscriber as neither a record nor silence. + [Test] + public async Task ALoadThatLandsAVariantNeverReachesASubscriberAsARecordAndTheSubscriberIsToldByKey() + { + var registry = CultDocumentRegistry.Shared; + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-variant-live-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, "wire.cc"); + using (var writer = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + writer.Commit(batch => batch.Upsert(typeof(VariantWireFixture), new VariantWireFixture { Name = "base", Power = 1 }, BaseKey)); + writer.UpsertVariantAsync(VariantKey, BaseKey, new[] + { + writer.Override(nameof(VariantWireFixture.Name), "big"), + writer.Override(nameof(VariantWireFixture.Power), 9) + }).GetAwaiter().GetResult(); + } + + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "variant-live-server", + Socket = socket + }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("variant-live-client"); + var snapshot = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var changes = new ConcurrentQueue(); + var errors = new ConcurrentQueue(); + client.OnCultNet(_ => snapshot.TrySetResult(true)); + client.OnCultNet(changes.Enqueue); + client.OnCultNet(errors.Enqueue); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + await WaitUntilAsync(() => client.Connected); + client.SendCultNet(new CultNetDatabaseSubscribeMessage + { + MessageId = "subscribe-all", + SubscriptionId = "all", + IncludeSnapshot = true + }); + await WaitUntilAsync(() => snapshot.Task.IsCompleted); + + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + + await WaitUntilAsync(() => !errors.IsEmpty || changes.Any(change => change.Document?.RecordKey == VariantKey.Value)); + await Task.Delay(200); // let anything else already in flight arrive + + Assert.That(changes.Select(change => change.Document?.RecordKey), Does.Not.Contain(VariantKey.Value), + "a variant must not reach a subscriber as a plain record"); + Assert.That(errors.Select(error => error.Error), Has.Some.Contains(VariantKey.Value), + "the subscriber is told, naming the variant"); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Test] + public async Task TheDatabaseServersLiveChangeForAVariantIsRefusedNamingIt() + { + var (cache, documents) = await BuildFixtureAsync(withVariant: true); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var schemaId = cache.Registry.GetRequired().SchemaId; + var change = new CultNetDatabaseChange( + CultNetDatabaseChangeKind.Added, + VariantKey, + schemaId, + database.Shards[0], + cache.Get(VariantKey)!, + previousDocument: null); + var method = typeof(CultNetDatabaseServer).GetMethod("CreateChangeMessage", BindingFlags.Instance | BindingFlags.NonPublic)!; + + var refused = Assert.Throws(() => method.Invoke(databaseServer, new object[] + { + change, "sub-1", new CultNetSelection { Projection = CultNetSelectionProjections.Document } + }))!; + + Assert.That(refused.InnerException, Is.TypeOf()); + Assert.That(refused.InnerException!.Message, Does.Contain(VariantKey.Value).And.Contain(BaseKey.Value)); + } + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + Assert.Fail("Condition was not satisfied before timeout."); + await Task.Delay(5); + } + } + + [CultDocument("variants.wire-fixture", "variants.wire-fixture.v1")] [MessagePackObject(AllowPrivate = true)] public sealed class VariantWireFixture From c66d79e248c17502b090374b145e0d6a77ae6c55 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 06:58:51 +0200 Subject: [PATCH 07/16] Variant overrides follow the plain-record outcome; bad writes and shared indexes are refused F3: an override drifts exactly as the same member would on a plain record. A slot the type dropped is ignored and reported by the store's schema resolution (LastSchemaMigrationReports), the one surface a plain record uses; a value that no longer decodes refuses the load, naming the variant and member. The second report surface (VariantMigrationReports, the per-variant report map, the catalog slot-name plumbing) is deleted. F4: an override for a slot the type does not have, or a value that does not decode as the member, is refused at write. Drift is for type changes after the fact, not for bad writes. F5: two overrides of one slot are refused. F2: a variant that shares an indexed value with another record is refused, naming both keys and the index (the R6 rule extended to [CultIndex]); plain record duplicate handling is unchanged. F7: a flatten now resolves against the same batch, so a base edited in that batch is in the flattened record; and the directory store refuses to load a variant page as it refuses to write one. The C1 vector's big variant now overrides its indexed code, as the unique-index rule requires. Co-Authored-By: Claude Sonnet 5 --- .../DirectoryMessagePackBackingStore.cs | 4 + .../cultcache-schema-compatibility.md | 18 +- src/GameCult.Caching/CultCache.cs | 156 ++++++------ src/GameCult.Caching/CultVariants.cs | 4 - tests/GameCult.Caching.Tests/VariantTests.cs | 222 ++++++++++++++++-- .../vectors/document-variants-c0/generate.py | 8 +- 6 files changed, 304 insertions(+), 108 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index fa9d4899..3ef36be2 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -502,6 +502,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(pagePayload); if (!string.Equals(record.Key, metadata.Key, StringComparison.Ordinal)) throw new InvalidDataException($"Record page '{path}' contains key '{record.Key}', expected '{metadata.Key}'."); + // The shared decoder reads a variant page; this store does not hold variants, so a load refuses it as a write does. + if (record.Variant != null) + throw new NotSupportedException( + $"Record '{record.Key}' is a variant of '{record.Variant.BaseKey}'; the directory store does not hold document variants yet."); return record; } diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index 818840ee..9ae6f553 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -108,7 +108,17 @@ This is not negotiable. Better a loud refusal than quiet bit-rot. ## Variant overrides -A variant's override drifts exactly as the same member would on a plain record: an -override naming a slot the type no longer has, or holding a value that no longer decodes -into the member's type, is ignored with a warning naming the variant and the member, and -the variant inherits the base value. There is no separate rule for variants. +A variant's override follows the same outcome as the same member on a plain record; there +is no separate rule for variants, and no separate report. + +- An override naming a slot the type no longer has is ignored, exactly as a plain record + ignores a dropped member. The drift is reported once, by the store's schema resolution + (`LastSchemaMigrationReports`), for the variant's record as for any other record of the + schema. The variant inherits the base value. +- An override holding a value that no longer decodes into the member's type refuses the + load, exactly as a plain record holding such a value does; the refusal names the variant + and the member. + +Drift is for type changes after the fact. A write is validated: an override for a slot the +type does not have, a value that does not decode as the member, or two overrides of one +slot is refused when written. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 00b849a0..f17d5b93 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1802,10 +1802,8 @@ namespace GameCult.Caching private readonly Dictionary<(Type Type, string Alias), Dictionary> _indexes = new(); private readonly Dictionary _globals = new(); private readonly CultCodec? _codec; - // Keys holding a variant, and the drift report of each variant whose overrides were partly ignored: both derived - // from the entries by Apply, never decided anywhere else. + // Keys holding a variant: derived from the entries by Apply, never decided anywhere else. private readonly HashSet _variantKeys = new(StringComparer.Ordinal); - private readonly Dictionary _variantReports = new(StringComparer.Ordinal); private readonly ConditionalWeakTable _handles = new(); private readonly Subject _changes = new(); private readonly object _gate = new(); @@ -1826,17 +1824,6 @@ namespace GameCult.Caching public CultCodec? Codec => _codec; - // Overrides a variant carries that no longer resolve (a slot the type dropped, a value that no longer decodes), - // reported as compatible drift: the override is ignored and the variant inherits the base value. - public IReadOnlyList VariantMigrationReports - { - get - { - lock (_gate) - return _variantReports.Values.ToArray(); - } - } - // The whole override of one member, encoded with this cache's codec, ready for UpsertVariant. public CultVariantOverride Override(Type documentType, string member, object? value) { @@ -2467,9 +2454,6 @@ namespace GameCult.Caching // Variants this admission did not write whose base chain it changed: re-resolved, same delta, same storedAt. public List Dependents { get; } = new(); - - // The drift report of every variant resolved here; null when nothing was ignored. - public Dictionary Reports { get; } = new(StringComparer.Ordinal); } // Resolves every admitted variant and every variant whose base chain this admission changes, base first, against @@ -2512,7 +2496,9 @@ namespace GameCult.Caching } var changed = new HashSet(incoming.Keys.Concat(gone), StringComparer.Ordinal); - var toResolve = new List(admitted.Where(stored => stored.Variant != null).Select(stored => stored.Key.Value)); + // A flatten resolves like the variant it replaces, against the post-batch set, so a base edited in the same + // batch is in it; it lands as the plain record of that resolution. + var toResolve = new List(admitted.Where(stored => stored.Variant != null || stored.Flattens).Select(stored => stored.Key.Value)); foreach (var variantKey in _variantKeys) { if (incoming.ContainsKey(variantKey) || gone.Contains(variantKey)) @@ -2538,16 +2524,14 @@ namespace GameCult.Caching var pending = new HashSet(toResolve, StringComparer.Ordinal); var done = new Dictionary(StringComparer.Ordinal); - var reports = new Dictionary(StringComparer.Ordinal); var stack = new List(); + Dictionary<(Type Type, string Alias), Dictionary>? claimed = null; foreach (var key in toResolve) ResolveOne(key); - var plan = new VariantPlan(admitted.Select(stored => stored.Variant != null ? done[stored.Key.Value] : stored).ToList()); + var plan = new VariantPlan(admitted.Select(stored => done.TryGetValue(stored.Key.Value, out var resolved) ? resolved : stored).ToList()); foreach (var key in toResolve.Where(key => !incoming.ContainsKey(key))) plan.Dependents.Add(done[key]); - foreach (var pair in reports) - plan.Reports[pair.Key] = pair.Value; return plan; CultStoredDocument ResolveOne(string key) @@ -2559,9 +2543,11 @@ namespace GameCult.Caching throw CycleRefusal(stack.SkipWhile(entry => entry != key).Append(key)); stack.Add(key); - var delta = variant.Variant!; + // A flatten carries no delta of its own; the variant it replaces does. + var isVariant = variant.Variant != null; + var delta = variant.Variant ?? _entries[key].Variant!; var baseStored = Lookup(delta.BaseKey) ?? throw MissingBaseRefusal(key, delta.BaseKey, gone.Contains(delta.BaseKey)); - if (baseStored.Variant != null && pending.Contains(delta.BaseKey)) + if (pending.Contains(delta.BaseKey)) baseStored = ResolveOne(delta.BaseKey); var descriptor = variant.Descriptor; @@ -2575,25 +2561,25 @@ namespace GameCult.Caching var type = descriptor.DocumentType; var kept = new List>(); - var ignored = new List(); - var warnings = new List(); + var seen = new HashSet(); + // Bad writes are refused; drift is for type changes after the fact. A load, a re-resolution and a flatten + // carry what was already stored, so they follow the plain-record outcome instead. + var writing = source == null && isVariant && incoming.ContainsKey(key); foreach (var entry in delta.Overrides) { if (entry.Op != CultOverrideOp.Set || entry.Path.Count != 1 || entry.Path[0].ElementId.Length != 0 || entry.Id.Length != 0) throw new InvalidOperationException( $"Variant {key} carries {entry.Op} at {entry.PathText}; this runtime resolves top-level Set overrides only."); var slot = entry.Path[0].Slot; + if (!seen.Add(slot)) + throw new InvalidOperationException($"Variant {key} overrides slot {slot} more than once; a variant has one override per slot."); var member = descriptor.RichMembers.FirstOrDefault(candidate => candidate.Slot == slot); if (member == null) { - var name = delta.SlotNames != null && delta.SlotNames.TryGetValue(slot, out var persistedName) - ? $"'{persistedName}'" : "unnamed"; - ignored.Add(slot); - warnings.Add(new CultSchemaMigrationWarning - { - Code = "ignored_extra_slot", - Message = $"Variant {key} overrides slot {slot} ({name}), which {type.Name} no longer has; the override is ignored and the variant inherits the base value." - }); + if (writing) + throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have."); + // A slot the type dropped soft-drifts exactly as the same member does on a plain record: ignored, + // reported by the store's schema resolution (LastSchemaMigrationReports), the variant inherits the base. continue; } @@ -2603,14 +2589,10 @@ namespace GameCult.Caching } catch (Exception exception) { - ignored.Add(slot); - warnings.Add(new CultSchemaMigrationWarning - { - Code = "ignored_undecodable_override", - Message = $"Variant {key} overrides slot {slot} ('{member.Member.Name}') with a value that no longer decodes as {member.MemberType.Name} " + - $"({exception.GetBaseException().Message}); the override is ignored and the variant inherits the base value." - }); - continue; + // A value that no longer decodes refuses exactly as a plain record holding it does. + throw new InvalidOperationException( + $"Variant {key} overrides '{member.Member.Name}' (slot {slot}) with a value that does not decode as {member.MemberType.Name}: " + + exception.GetBaseException().Message, exception); } kept.Add(new KeyValuePair(slot, entry.Value)); @@ -2620,32 +2602,73 @@ namespace GameCult.Caching var document = _codec.Deserialize(type, type, kept.Count == 0 ? payload : _codec.Overlay(payload, kept)) ?? throw new InvalidOperationException($"Variant {key} resolved to nothing."); - // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. - if (descriptor.NameAccessor?.Invoke(document) is { Length: > 0 } variantName) + if (isVariant) { - for (var ancestor = baseStored; ancestor != null; - ancestor = ancestor.Variant == null ? null : done.TryGetValue(ancestor.Variant.BaseKey, out var above) ? above : Lookup(ancestor.Variant.BaseKey)) + // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. + if (descriptor.NameAccessor?.Invoke(document) is { Length: > 0 } variantName) { - if (descriptor.NameAccessor(ancestor.Document) == variantName) + for (var ancestor = baseStored; ancestor != null; + ancestor = ancestor.Variant == null ? null : done.TryGetValue(ancestor.Variant.BaseKey, out var above) ? above : Lookup(ancestor.Variant.BaseKey)) + { + if (descriptor.NameAccessor(ancestor.Document) == variantName) + throw new InvalidOperationException( + $"Variant {key} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); + } + } + + // The same rule for an indexed value: a unique index maps one value to one key, so a variant that + // inherited its base's value (or any other record's) would be found in place of it or lose it. + foreach (var pair in descriptor.IndexAccessors) + { + var value = pair.Value(document); + if (string.IsNullOrWhiteSpace(value)) + continue; + claimed ??= ClaimIncomingIndexValues(); + var claim = (type, pair.Key); + string? other = null; + if (claimed.TryGetValue(claim, out var claims) && claims.TryGetValue(value, out var claimant) && claimant != key) + other = claimant; + else if (_indexes.TryGetValue(claim, out var held) && held.TryGetValue(value, out var holder) && holder != key && + !incoming.ContainsKey(holder) && !gone.Contains(holder) && !pending.Contains(holder)) + other = holder; + if (other != null) throw new InvalidOperationException( - $"Variant {key} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); + $"Variant {key} and {other} both hold index {pair.Key} '{value}'; a variant must override its indexed member."); } } - reports[key] = ignored.Count == 0 - ? null - : new CultSchemaMigrationReport + if (descriptor.IndexAccessors.Count > 0) + { + claimed ??= ClaimIncomingIndexValues(); + foreach (var pair in descriptor.IndexAccessors) { - PersistedSchemaId = descriptor.SchemaId, - LocalSchemaId = descriptor.SchemaId, - PersistedSchemaName = descriptor.SchemaName, - LocalSchemaName = descriptor.SchemaName, - Kind = CultSchemaMigrationKind.CompatibleDrift, - IgnoredExtraSlots = ignored.OrderBy(slot => slot).ToArray(), - Warnings = warnings.ToArray() - }; + var value = pair.Value(document); + if (!string.IsNullOrWhiteSpace(value)) + MapOf(claimed, (type, pair.Key))[value] = key; + } + } + stack.RemoveAt(stack.Count - 1); - return done[key] = variant.Resolved(document); + return done[key] = isVariant + ? variant.Resolved(document) + : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { Flattens = true }; + } + + // The indexed values the plain records of this admission will hold, by (type, index) then value. + Dictionary<(Type Type, string Alias), Dictionary> ClaimIncomingIndexValues() + { + var claims = new Dictionary<(Type Type, string Alias), Dictionary>(); + foreach (var stored in incoming.Values.Where(stored => stored.Variant == null && !stored.Flattens)) + { + foreach (var pair in stored.Descriptor.IndexAccessors) + { + var value = pair.Value(stored.Document); + if (!string.IsNullOrWhiteSpace(value)) + MapOf(claims, (stored.Descriptor.DocumentType, pair.Key))[value] = stored.Key.Value; + } + } + + return claims; } } @@ -2672,7 +2695,6 @@ namespace GameCult.Caching _entries.Remove(stored.Key.Value); _handles.Remove(existing.Document); _variantKeys.Remove(stored.Key.Value); - _variantReports.Remove(stored.Key.Value); changes.Add(new Change(CultCacheDocumentChangeKind.Removed, existing, null, existing.Document, ++_sequence)); } @@ -2692,10 +2714,6 @@ namespace GameCult.Caching _variantKeys.Remove(stored.Key.Value); else _variantKeys.Add(stored.Key.Value); - if (plan.Reports.TryGetValue(stored.Key.Value, out var report) && report != null) - _variantReports[stored.Key.Value] = report; - else - _variantReports.Remove(stored.Key.Value); changes.Add(new Change( previous == null ? CultCacheDocumentChangeKind.Added : CultCacheDocumentChangeKind.Updated, stored, @@ -2911,16 +2929,12 @@ namespace GameCult.Caching var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); if (record.Variant != null) { - // The cache resolves it against its base when it admits the load; the catalog names the slots the type may have dropped. - var persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, record.SchemaId, StringComparison.Ordinal)); + // The cache resolves it against its base when it admits the load. return new CultStoredDocument( new CultRecordKey(record.Key), record.StoredAt, resolution.Descriptor, - new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides) - { - SlotNames = persisted?.Members.GroupBy(member => member.Slot).ToDictionary(group => group.Key, group => group.First().MemberName) - }, + new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides), null); } diff --git a/src/GameCult.Caching/CultVariants.cs b/src/GameCult.Caching/CultVariants.cs index 08882c79..ee5cc523 100644 --- a/src/GameCult.Caching/CultVariants.cs +++ b/src/GameCult.Caching/CultVariants.cs @@ -87,9 +87,5 @@ namespace GameCult.Caching public string BaseKey { get; } public IReadOnlyList Overrides { get; } - - // Member names by slot from the catalog the delta was loaded under, for warnings about slots the type dropped. - // Not persisted. - internal IReadOnlyDictionary? SlotNames { get; set; } } } diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index fec33c96..f8544e50 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -50,12 +50,13 @@ namespace GameCult.Caching.Tests private static void SeedBase(CultCache cache) => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Laser(), BaseKey)); - // laser-big: a variant of laser that overrides its name and power and inherits the rest. + // laser-big: a variant of laser that overrides its name, power and code (an indexed value must be its own) and inherits the rest. private static void SeedBig(CultCache cache, CultRecordKey? key = null, CultRecordKey? baseKey = null, string name = "laser big", int power = 25) => cache.Commit(batch => batch.UpsertVariant(key ?? BigKey, baseKey ?? BaseKey, new[] { cache.Override(nameof(VariantGear.Name), name), - cache.Override(nameof(VariantGear.Power), power) + cache.Override(nameof(VariantGear.Power), power), + cache.Override(nameof(VariantGear.Code), name) })); private static void EditBase(CultCache cache, Action edit) => @@ -85,7 +86,7 @@ namespace GameCult.Caching.Tests var big = cache.Get(BigKey)!; Assert.That(big.Name, Is.EqualTo("laser big")); Assert.That(big.Power, Is.EqualTo(25)); - Assert.That(big.Code, Is.EqualTo("l1"), "an inherited member"); + Assert.That(big.Code, Is.EqualTo("laser big"), "the override the unique index requires"); Assert.That(big.Tags, Is.EqualTo(new[] { "beam" }), "an inherited list"); Assert.That(cache.Get(BaseKey)!.Power, Is.EqualTo(10)); Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); @@ -133,9 +134,10 @@ namespace GameCult.Caching.Tests Assert.That(cache.GetByName("laser big")!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); Assert.That(cache.GetByName("laser big")!.Power, Is.EqualTo(25), "the override survives the base edit"); Assert.That(cache.GetByIndex("code", "l9")!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); - // big inherited the old code; its stale index entry must be gone, not shadowing. + // the base's old code entry must be gone, not shadowing, and its new one must find the base itself. Assert.That(cache.GetByIndex("code", "l1"), Is.Null); - Assert.That(cache.GetByIndex("code", "l2")!.Name, Is.Not.Null); + Assert.That(cache.GetByIndex("code", "l2")!.Name, Is.EqualTo("laser")); + Assert.That(cache.GetByIndex("code", "laser big")!.Name, Is.EqualTo("laser big")); Assert.That(seen.Select(entry => entry.Key), Is.EquivalentTo(new[] { BaseKey.Value, BigKey.Value, "laser-coded" })); Assert.That(seen.Select(entry => entry.Sequence), Is.EqualTo(Enumerable.Range((int)seen[0].Sequence, seen.Count).Select(n => (long)n)), @@ -297,7 +299,7 @@ namespace GameCult.Caching.Tests batch.UpsertVariant(BigKey, spare, cache.GetStored(BigKey)!.Variant!.Overrides); batch.Remove(BaseKey); }), Is.True, "rebased in the same batch"); - Assert.That(cache.Get(BigKey)!.Code, Is.EqualTo(string.Empty), "now resolved from spare"); + Assert.That(cache.Get(BigKey)!.Tags, Is.Empty, "now resolved from spare"); Assert.That(cache.Commit(batch => { @@ -435,36 +437,204 @@ namespace GameCult.Caching.Tests Assert.That(variant.Power, Is.EqualTo(7), "the override the type still has applies"); Assert.That(variant.Name, Is.EqualTo("drift variant")); - var report = cache.VariantMigrationReports.Single(); - Assert.That(report.Kind, Is.EqualTo(CultSchemaMigrationKind.CompatibleDrift)); - Assert.That(report.IgnoredExtraSlots, Is.EqualTo(new[] { 2 })); - var warning = report.Warnings.Single(); - Assert.That(warning.Code, Is.EqualTo("ignored_extra_slot"), "the code a dropped member gets on a plain record"); - Assert.That(warning.Message, Does.Contain("drift-variant").And.Contain("Wing")); + // The same decision, on the same surface, that the plain base record gets from the store's schema resolution. + var reports = cache.BackingStores[0].LastSchemaMigrationReports; + Assert.That(reports, Has.Count.EqualTo(2), "one per record: the base and the variant"); + Assert.That(reports.Select(report => report.Kind), Is.All.EqualTo(CultSchemaMigrationKind.CompatibleDrift)); + Assert.That(reports.Select(report => string.Join(",", report.IgnoredExtraSlots)), Is.All.EqualTo("2")); + Assert.That(reports.SelectMany(report => report.Warnings).Select(warning => warning.Code), Is.All.EqualTo("ignored_extra_slot")); + } + + private static readonly byte[] NotAnInt = { 0xa3, (byte)'a', (byte)'b', (byte)'c' }; + + private static CultVariantOverride TextOverride(int slot, string value) + { + var bytes = new byte[value.Length + 1]; + bytes[0] = (byte)(0xa0 | value.Length); + for (var index = 0; index < value.Length; index++) + bytes[index + 1] = (byte)value[index]; + return CultVariantOverride.Set(slot, bytes); + } + + private static CultPersistedRecord OverridingRecord(string key, string baseKey, params CultVariantOverride[] overrides) => new() + { + Key = key, + SchemaId = Registry.GetRequired().SchemaId, + StoredAt = "2026-01-01T00:00:00.0000000Z", + Payload = Array.Empty(), + Variant = new CultVariantDelta(baseKey, overrides) + }; + + [Test] + public void AnOverrideThatNoLongerDecodesRefusesTheLoadAsAPlainRecordHoldingItDoes() + { + // VariantGear.Power is [Key(1)] int; a string there is the nested-type-change case with the schema id unchanged. + var plainRecord = new CultPersistedRecord + { + Key = "hand-plain", + SchemaId = Registry.GetRequired().SchemaId, + StoredAt = "2026-01-01T00:00:00.0000000Z", + Payload = new byte[] { 0x94, 0xa1, (byte)'x', 0xa3, (byte)'a', (byte)'b', (byte)'c', 0xa0, 0x90 } + }; + Assert.That(Assert.Catch(() => Open(WriteStore(plainRecord)).Dispose()), Is.Not.Null, "the plain-record outcome is a refusal"); + + var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", CultVariantOverride.Set(1, NotAnInt))); + var refused = Assert.Catch(() => Open(path).Dispose())!; + Assert.That(refused.Message, Does.Contain("hand-variant").And.Contain("Power")); } [Test] - public void AnOverrideThatNoLongerDecodesIsIgnoredWithAWarningAndTheBaseValueApplies() + public void AnOverrideOfASlotTheTypeNeverHadIsIgnoredOnLoadAsAPlainRecordIgnoresAnExtraSlot() + { + var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", + CultVariantOverride.Set(99, new byte[] { 0x01 }), + TextOverride(0, "hand variant"), + TextOverride(2, "hand-code"))); + using var cache = Open(path); + Assert.That(cache.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + Assert.That(cache.Get(new CultRecordKey("hand-variant"))!.Power, Is.EqualTo(10), "inherits the base"); + } + + // ---- a bad write is refused; drift is for type changes after the fact ---- + + [Test] + public void AnInvalidOverrideIsRefusedAtWriteNamingVariantSlotAndReasonAndNothingLands() + { + var path = PathOf("gear.cc"); + using var cache = Open(path); + SeedBase(cache); + + var unknown = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + CultVariantOverride.Set(99, new byte[] { 0x01 }) + }))); + Assert.That(unknown.Message, Does.Contain(BigKey.Value).And.Contain("99").And.Contain("does not have")); + + var wrongType = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey, new[] + { + CultVariantOverride.Set(1, NotAnInt) + }).GetAwaiter().GetResult()); + Assert.That(wrongType.Message, Does.Contain(BigKey.Value).And.Contain("Power").And.Contain("slot 1")); + + Assert.That(cache.Get(BigKey), Is.Null); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "no variant was persisted"); + } + + [Test] + public void TwoOverridesOfOneSlotAreRefusedAtWriteAndAnUnknownMemberNameIsAnArgumentException() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + + var twice = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Power), 1), + cache.Override(nameof(VariantGear.Power), 2) + }))); + Assert.That(twice.Message, Does.Contain(BigKey.Value).And.Contain("slot 1").And.Contain("more than once")); + Assert.That(cache.Get(BigKey), Is.Null); + + var unknownName = Assert.Throws(() => cache.Override("Nope", 1))!; + Assert.That(unknownName.Message, Does.Contain("Nope")); + } + + // ---- a variant may not share an indexed value with another record ---- + + [Test] + public void AVariantThatInheritsAnIndexedValueIsRefusedNamingBothKeysAndTheIndex() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + + var inherited = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big") + }))); + Assert.That(inherited.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("code").And.Contain("l1")); + + var explicitly = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big"), + cache.Override(nameof(VariantGear.Code), "l1") + }))); + Assert.That(explicitly.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value)); + Assert.That(cache.Get(BigKey), Is.Null, "nothing landed"); + Assert.That(cache.GetByIndex("code", "l1")!.Name, Is.EqualTo("laser")); + + SeedBig(cache); + Assert.That(cache.GetByIndex("code", "l1")!.Name, Is.EqualTo("laser")); + Assert.That(cache.GetByIndex("code", "laser big")!.Name, Is.EqualTo("laser big")); + } + + [Test] + public void ABaseEditThatWouldGiveItsVariantsIndexedValueToTheBaseIsRefused() + { + using var cache = Open(PathOf("gear.cc")); + SeedBase(cache); + SeedBig(cache); + + var refused = Refused(() => EditBase(cache, gear => gear.Code = "laser big")); + Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("code")); + Assert.That(cache.Get(BaseKey)!.Code, Is.EqualTo("l1"), "nothing landed"); + } + + [Test] + public void ALoadedVariantThatInheritsAnIndexedValueRefusesTheLoadNamingBothKeys() + { + var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base")); + var refused = Assert.Throws(() => Open(path).Dispose())!; + Assert.That(refused.Message, Does.Contain("hand-variant").And.Contain("hand-base").And.Contain("code")); + } + + // ---- flatten resolves against the same batch ---- + + [Test] + public void AFlattenAndABaseEditInOneBatchLandTheResolutionOfTheWholeBatch() { var path = PathOf("gear.cc"); - const int notAnInt = 1; // VariantGear.Power, [Key(1)] using (var cache = Open(path)) { SeedBase(cache); - cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + SeedBig(cache); + cache.Commit(batch => { - cache.Override(nameof(VariantGear.Name), "laser big"), - CultVariantOverride.Set(notAnInt, new byte[] { 0xa3, (byte)'a', (byte)'b', (byte)'c' }) - })); - var warning = cache.VariantMigrationReports.Single().Warnings.Single(); - Assert.That(warning.Message, Does.Contain(BigKey.Value).And.Contain("Power")); - Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(10), "inherits the base value"); - Assert.That(cache.Get(BigKey)!.Name, Is.EqualTo("laser big")); + var edited = Laser(); + edited.Tags.Add("pierce"); + batch.Upsert(typeof(VariantGear), edited, BaseKey); + batch.Flatten(BigKey); + }); + + Assert.That(cache.GetStored(BigKey)!.Variant, Is.Null); + Assert.That(cache.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" }), "the edit made in the same batch"); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25)); } using var reloaded = Open(path); - Assert.That(reloaded.Get(BigKey)!.Power, Is.EqualTo(10)); - Assert.That(reloaded.VariantMigrationReports.Single().Warnings.Single().Message, Does.Contain("Power")); + Assert.That(reloaded.GetStored(BigKey)!.Variant, Is.Null); + Assert.That(reloaded.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); + } + + // ---- the directory store refuses a variant page on load as it does on write ---- + + [Test] + public void TheDirectoryStoreRefusesToLoadAVariantPage() + { + var manifest = PathOf("dir-page.cc"); + var records = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(manifest); + Directory.CreateDirectory(records); + var page = CultDocumentMessagePackSerialization.SerializePersistedRecord(OverridingRecord("hand-variant", "hand-base", TextOverride(0, "x"))); + var hash = System.Security.Cryptography.SHA256.HashData(page); + File.WriteAllBytes(Path.Combine(records, Convert.ToHexString(hash).ToLowerInvariant() + ".msgpack"), page); + + using var store = new DirectoryMessagePackBackingStore(manifest, records); + var read = typeof(DirectoryMessagePackBackingStore).GetMethod( + "ReadPersistedRecordPage", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!; + var refused = Assert.Throws( + () => read.Invoke(store, new object?[] { new CultPersistedRecord { Key = "hand-variant", Payload = hash }, null }))!; + + Assert.That(refused.InnerException, Is.TypeOf()); + Assert.That(refused.InnerException!.Message, Does.Contain("hand-variant").And.Contain("directory store")); } // ---- persisted format ---- @@ -541,7 +711,7 @@ namespace GameCult.Caching.Tests using var readOnly = Open(path, readOnly: true); Assert.That(readOnly.Get(BigKey)!.Power, Is.EqualTo(25)); - Assert.That(readOnly.Get(BigKey)!.Code, Is.EqualTo("l1")); + Assert.That(readOnly.Get(BigKey)!.Code, Is.EqualTo("laser big")); // A base edit another writer commits reaches the read-only cache's variant when it pulls: a re-resolution is no write. using var writer = Open(path); @@ -705,7 +875,7 @@ namespace GameCult.Caching.Tests using (var cache = Open(path)) { Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25)); - Assert.That(cache.Get(BigKey)!.Code, Is.EqualTo("l1")); + Assert.That(cache.Get(BigKey)!.Code, Is.EqualTo("laser big")); cache.BackingStores[0].PushAll(); } diff --git a/tests/vectors/document-variants-c0/generate.py b/tests/vectors/document-variants-c0/generate.py index e3d2547d..c6e5d2da 100644 --- a/tests/vectors/document-variants-c0/generate.py +++ b/tests/vectors/document-variants-c0/generate.py @@ -11,9 +11,11 @@ at test time. Run: python generate.py (needs msgpack). v1-base.msgpack NOT generated here: written once by the Python runtime at CultLib 69a21bb (SingleFileMessagePackBackingStore.push of alpha and beta, type vectors.item, before any C0 code existed) - variant-v2.msgpack what C1 writes for a variant: header v2, record slot 4 = - [baseKey, overrides[]], empty payload (override op Set = 0, - path steps are [slot, elementId]) + variant-v2.msgpack the wire shape of a variant record, hand-built: header v2, record slot 4 = + [baseKey, overrides[]], empty payload (override op Set = 0, path steps + are [slot, elementId]). C1 does not write this store and refuses to load + it: the variant resolves to its base's name (R6). It exercises readers + that refuse a variant by version or record. """ import pathlib From 2845d3591bee3e4683f14b08dbb9aba12bfd7992 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 06:59:29 +0200 Subject: [PATCH 08/16] Regenerate the C1 variant vector: the variant overrides its indexed code The unique-index rule refuses a variant that inherits its base's [CultIndex] value, so the vector's laser-big now carries a code override. Written by the code under test through the production path; the other runtimes still refuse it by version and by the laser-big key. Co-Authored-By: Claude Sonnet 5 --- .../variant-store.msgpack | Bin 1391 -> 1408 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/vectors/document-variants-c1/variant-store.msgpack b/tests/vectors/document-variants-c1/variant-store.msgpack index 3604a68da1979eac264453f272960bea7d7ed467..692b225d2c8591a97a6cbd625093df89e6085d29 100644 GIT binary patch delta 101 zcmaFQ)xf=BEepSim8qqbp{bsUiIJhXx#8qJEY+ScF;jD66GH<-Z36=<1A`+Bla}Tr i7N-_Xp29G362pQ8E5SU4q|9`X0ONuMk|3H1Sponi5+WV| delta 84 zcmZqRe$TaGEepS)m5I5Pfu){-p^1U1x#i?NEY%h;F+)=mb5nCuZ36=<1A`+Bla}Tr d7N-_Xn!+$~62pQ8E5SU4q|9`X0ONuMk^mHQ8wdaZ From 5d6847e2208bf60b52d1f94fc1adfb8517ebd6bd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:04:01 +0200 Subject: [PATCH 09/16] Align the C1 tests with the unique-index rule and fix the live-subscription fixture Hand-written and directory-store variants now override their indexed value or name as the rules require; the subscription test writes its variant through a committed batch so the file holds it, subscribes by key, and reports what it saw when it times out. Co-Authored-By: Claude Sonnet 5 --- tests/GameCult.Caching.Tests/VariantTests.cs | 8 ++++---- .../CultNetDocumentRegistryVariantTests.cs | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index f8544e50..2cd7cde2 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -90,7 +90,7 @@ namespace GameCult.Caching.Tests Assert.That(big.Tags, Is.EqualTo(new[] { "beam" }), "an inherited list"); Assert.That(cache.Get(BaseKey)!.Power, Is.EqualTo(10)); Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); - Assert.That(cache.GetStored(BigKey)!.Variant!.Overrides, Has.Count.EqualTo(2)); + Assert.That(cache.GetStored(BigKey)!.Variant!.Overrides, Has.Count.EqualTo(3)); } [Test] @@ -442,7 +442,7 @@ namespace GameCult.Caching.Tests Assert.That(reports, Has.Count.EqualTo(2), "one per record: the base and the variant"); Assert.That(reports.Select(report => report.Kind), Is.All.EqualTo(CultSchemaMigrationKind.CompatibleDrift)); Assert.That(reports.Select(report => string.Join(",", report.IgnoredExtraSlots)), Is.All.EqualTo("2")); - Assert.That(reports.SelectMany(report => report.Warnings).Select(warning => warning.Code), Is.All.EqualTo("ignored_extra_slot")); + Assert.That(reports.Select(report => report.Warnings.Select(warning => warning.Code)), Has.All.Contain("ignored_extra_slot")); } private static readonly byte[] NotAnInt = { 0xa3, (byte)'a', (byte)'b', (byte)'c' }; @@ -582,7 +582,7 @@ namespace GameCult.Caching.Tests [Test] public void ALoadedVariantThatInheritsAnIndexedValueRefusesTheLoadNamingBothKeys() { - var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base")); + var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", TextOverride(0, "hand variant"))); var refused = Assert.Throws(() => Open(path).Dispose())!; Assert.That(refused.Message, Does.Contain("hand-variant").And.Contain("hand-base").And.Contain("code")); } @@ -843,7 +843,7 @@ namespace GameCult.Caching.Tests SeedBase(cache); var push = Assert.Throws(() => cache.UpsertVariantAsync( - BigKey, BaseKey, new[] { cache.Override(nameof(VariantGear.Name), "laser big") }).GetAwaiter().GetResult())!; + BigKey, BaseKey, new[] { cache.Override(nameof(VariantGear.Name), "laser big"), cache.Override(nameof(VariantGear.Code), "laser big") }).GetAwaiter().GetResult())!; Assert.That(push.Message, Does.Contain(BigKey.Value)); var commit = Assert.Throws(() => SeedBig(cache))!; Assert.That(commit.Message, Does.Contain(BigKey.Value)); diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 2a8ddc58..8c70c31e 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -83,11 +83,11 @@ namespace GameCult.Networking.Tests using (var writer = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) { writer.Commit(batch => batch.Upsert(typeof(VariantWireFixture), new VariantWireFixture { Name = "base", Power = 1 }, BaseKey)); - writer.UpsertVariantAsync(VariantKey, BaseKey, new[] + writer.Commit(batch => batch.UpsertVariant(VariantKey, BaseKey, new[] { writer.Override(nameof(VariantWireFixture.Name), "big"), writer.Override(nameof(VariantWireFixture.Power), 9) - }).GetAwaiter().GetResult(); + })); } var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); @@ -128,13 +128,15 @@ namespace GameCult.Networking.Tests { MessageId = "subscribe-all", SubscriptionId = "all", + RecordKeys = new[] { BaseKey.Value, VariantKey.Value }, IncludeSnapshot = true }); await WaitUntilAsync(() => snapshot.Task.IsCompleted); cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); - await WaitUntilAsync(() => !errors.IsEmpty || changes.Any(change => change.Document?.RecordKey == VariantKey.Value)); + await WaitUntilAsync(() => !errors.IsEmpty || changes.Any(change => change.Document?.RecordKey == VariantKey.Value), + () => $"changes={string.Join(",", changes.Select(c => c.ChangeKind + ":" + c.Document?.RecordKey))} errors={string.Join(",", errors.Select(e => e.Error))} cacheKeys={cache.GetStored(BaseKey) != null}/{cache.GetStored(VariantKey) != null}"); await Task.Delay(200); // let anything else already in flight arrive Assert.That(changes.Select(change => change.Document?.RecordKey), Does.Not.Contain(VariantKey.Value), @@ -180,13 +182,13 @@ namespace GameCult.Networking.Tests Assert.That(refused.InnerException!.Message, Does.Contain(VariantKey.Value).And.Contain(BaseKey.Value)); } - private static async Task WaitUntilAsync(Func condition) + private static async Task WaitUntilAsync(Func condition, Func? diagnostic = null) { var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); while (!condition()) { if (DateTimeOffset.UtcNow > deadline) - Assert.Fail("Condition was not satisfied before timeout."); + Assert.Fail("Condition was not satisfied before timeout. " + diagnostic?.Invoke()); await Task.Delay(5); } } From 354f9743e6524be706b4f57a86c0d3278ea3edae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:28:24 +0200 Subject: [PATCH 10/16] Refuse a plain write that shares a variant's indexed value; a flush sheds overrides the type lacks The index rule is about the commit's result, so it now runs for every commit that changes an indexed value: a plain record landing on a value a variant of the post-commit set holds is refused naming both keys and the index (the held index is never trusted; a plain write had already overwritten it). Plain-vs-plain is untouched. A variant sheds overrides of slots its type lacks when it is flushed, as a plain record sheds the member, and the load's report names the slot when the schema itself never had it. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 98 +++++++++++- tests/GameCult.Caching.Tests/VariantTests.cs | 150 +++++++++++++++++++ 2 files changed, 245 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index f17d5b93..2d4e7137 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2520,7 +2520,11 @@ namespace GameCult.Caching } if (toResolve.Count == 0) - return new VariantPlan(admitted); + { + var unchanged = new VariantPlan(admitted); + RefusePlainSharingWithVariant(unchanged, incoming, gone); + return unchanged; + } var pending = new HashSet(toResolve, StringComparer.Ordinal); var done = new Dictionary(StringComparer.Ordinal); @@ -2532,6 +2536,7 @@ namespace GameCult.Caching var plan = new VariantPlan(admitted.Select(stored => done.TryGetValue(stored.Key.Value, out var resolved) ? resolved : stored).ToList()); foreach (var key in toResolve.Where(key => !incoming.ContainsKey(key))) plan.Dependents.Add(done[key]); + RefusePlainSharingWithVariant(plan, incoming, gone); return plan; CultStoredDocument ResolveOne(string key) @@ -2672,6 +2677,61 @@ namespace GameCult.Caching } } + // The index rule is about the commit's result, so it holds for every commit that changes an indexed value, in either + // order: a variant resolving onto another record's value is refused in ResolveOne; a plain record landing on a + // variant's value is refused here, against the variants of the post-commit set (never the held index, which the plain + // write would already have overwritten). Plain-vs-plain sharing is not this rule's business. + private void RefusePlainSharingWithVariant( + VariantPlan plan, + IReadOnlyDictionary incoming, + HashSet gone) + { + Dictionary<(Type Type, string Alias), Dictionary>? variantValues = null; + foreach (var stored in plan.Admitted) + { + if (stored.Variant != null || stored.Descriptor.IndexAccessors.Count == 0) + continue; + foreach (var pair in stored.Descriptor.IndexAccessors) + { + var value = pair.Value(stored.Document); + if (string.IsNullOrWhiteSpace(value)) + continue; + variantValues ??= HeldByVariants(plan, incoming, gone); + if (variantValues.TryGetValue((stored.Descriptor.DocumentType, pair.Key), out var values) && + values.TryGetValue(value, out var variantKey) && variantKey != stored.Key.Value) + throw new InvalidOperationException( + $"Record {stored.Key.Value} and variant {variantKey} both hold index {pair.Key} '{value}'; " + + "a variant may not share an indexed value with another record."); + } + } + } + + // The indexed values the variants of the post-commit set hold: those this commit resolved, and those it left alone. + private Dictionary<(Type Type, string Alias), Dictionary> HeldByVariants( + VariantPlan plan, + IReadOnlyDictionary incoming, + HashSet gone) + { + var variants = new Dictionary(StringComparer.Ordinal); + foreach (var variantKey in _variantKeys.Where(variantKey => !incoming.ContainsKey(variantKey) && !gone.Contains(variantKey))) + variants[variantKey] = _entries[variantKey]; + foreach (var stored in plan.Admitted.Concat(plan.Dependents).Where(stored => stored.Variant != null)) + variants[stored.Key.Value] = stored; + + var held = new Dictionary<(Type Type, string Alias), Dictionary>(); + foreach (var stored in variants.Values) + { + foreach (var pair in stored.Descriptor.IndexAccessors) + { + var value = pair.Value(stored.Document); + if (!string.IsNullOrWhiteSpace(value)) + MapOf(held, (stored.Descriptor.DocumentType, pair.Key))[value] = stored.Key.Value; + } + } + + return held; + } + private static InvalidOperationException CycleRefusal(IEnumerable keys) => new($"Variant bases form a cycle: {string.Join(" -> ", keys)}."); @@ -2917,10 +2977,41 @@ namespace GameCult.Caching SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt, Payload = entry.Variant != null ? Array.Empty() : serializePayload(entry.Document), - Variant = entry.Variant == null ? null : new CultVariantDelta(entry.Variant.BaseKey, entry.Variant.Overrides) + Variant = entry.Variant == null ? null : new CultVariantDelta(entry.Variant.BaseKey, OverridesTheTypeHas(entry).ToArray()) }; } + // A plain record sheds a member its type dropped when it is flushed; a variant sheds the override of that slot, so + // a later type reusing the slot number cannot resurrect it. The load's report names the shed slots. + private static IEnumerable OverridesTheTypeHas(CultStoredDocument entry) => + entry.Variant!.Overrides.Where(entryOverride => + entryOverride.Path.Count != 1 || entry.Descriptor.RichMembers.Any(member => member.Slot == entryOverride.Path[0].Slot)); + + // A variant's override of a slot its type lacks is ignored like an extra slot on a plain record. Schema resolution + // reports slots the persisted schema had and the type dropped; an override the schema never had is added here. + protected static CultSchemaMigrationReport ReportVariantSheds(CultSchemaMigrationReport report, CultStoredDocument stored) + { + if (stored.Variant == null) + return report; + var shed = stored.Variant.Overrides + .Where(entryOverride => entryOverride.Path.Count == 1 && + stored.Descriptor.RichMembers.All(member => member.Slot != entryOverride.Path[0].Slot)) + .Select(entryOverride => entryOverride.Path[0].Slot) + .Except(report.IgnoredExtraSlots) + .Distinct() + .OrderBy(slot => slot) + .ToArray(); + if (shed.Length == 0) + return report; + report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(shed).OrderBy(slot => slot).ToArray(); + report.Warnings = report.Warnings.Concat(shed.Select(slot => new CultSchemaMigrationWarning + { + Code = "ignored_extra_slot", + Message = $"Variant {stored.Key.Value} overrides slot {slot}, which {stored.Descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush." + })).ToArray(); + return report; + } + protected CultStoredDocument ToStoredDocument( CultPersistedRecord record, IReadOnlyCollection catalog, @@ -2993,8 +3084,9 @@ namespace GameCult.Caching var persisted = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { - reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog)); + var report = Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog); var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload); + reports.Add(ReportVariantSheds(report, stored)); persisted[stored.Key.Value] = stored; } diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 2cd7cde2..c81e892b 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -587,6 +587,140 @@ namespace GameCult.Caching.Tests Assert.That(refused.Message, Does.Contain("hand-variant").And.Contain("hand-base").And.Contain("code")); } + // ---- the index rule holds for the commit's result, in either write order, and the store always reopens ---- + + private static VariantGear Other(string code) => new() { Name = "other", Power = 1, Code = code }; + + [Test] + public async Task APlainWriteThatWouldShareAVariantsIndexedValueIsRefusedNamingBothKeysAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + using (var cache = Open(path)) + { + SeedBase(cache); + SeedBig(cache); + + var refused = Refused(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("laser big"), other))); + Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("laser big")); + + var single = Assert.ThrowsAsync( + () => cache.UpsertAsync(Other("laser big"), new CultRecordHandle(other)))!; + Assert.That(single.Message, Does.Contain(other.Value).And.Contain(BigKey.Value)); + + Assert.That(cache.Get(other), Is.Null, "nothing landed"); + Assert.That(cache.GetByIndex("code", "laser big")!.Name, Is.EqualTo("laser big"), "the variant is still found by its value"); + EditBase(cache, gear => gear.Power = 11); + Assert.That(cache.Get(BigKey)!.Power, Is.EqualTo(25), "the variant is still editable through its base"); + await Task.CompletedTask; + } + + using var reopened = Open(path); + Assert.That(reopened.GetByIndex("code", "laser big")!.Name, Is.EqualTo("laser big")); + Assert.That(reopened.Get(other), Is.Null); + } + + [Test] + public void ABaseEditOrANewPlainRecordFirstThenAVariantOnTheSameValueIsRefusedAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + using (var cache = Open(path)) + { + SeedBase(cache); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("laser big"), other)); + + var refused = Refused(() => SeedBig(cache)); + Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain(other.Value).And.Contain("code").And.Contain("laser big")); + Assert.That(cache.Get(BigKey), Is.Null, "nothing landed"); + + // The same batch, either staging order: the result is what is judged. + var sameBatchPlainFirst = Refused(() => cache.Commit(batch => + { + batch.Upsert(typeof(VariantGear), Other("taken"), new CultRecordKey("taker")); + batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big"), + cache.Override(nameof(VariantGear.Code), "taken") + }); + })); + Assert.That(sameBatchPlainFirst.Message, Does.Contain(BigKey.Value).And.Contain("taker")); + var sameBatchVariantFirst = Refused(() => cache.Commit(batch => + { + batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big"), + cache.Override(nameof(VariantGear.Code), "taken") + }); + batch.Upsert(typeof(VariantGear), Other("taken"), new CultRecordKey("taker")); + })); + Assert.That(sameBatchVariantFirst.Message, Does.Contain(BigKey.Value).And.Contain("taker")); + + // Plain-vs-plain sharing is not this rule's business. + Assert.DoesNotThrow(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("l1"), new CultRecordKey("twin")))); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(BigKey), Is.Null); + Assert.That(reopened.Get(other), Is.Not.Null); + } + + // ---- a variant sheds overrides its type lacks when it is flushed, as a plain record sheds the member ---- + + [Test] + public void AFlushShedsAnOverrideOfADroppedSlotSoALaterTypeReusingTheSlotDoesNotResurrectIt() + { + var path = PathOf("drift.cc"); + var wide = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }); + using (var writer = Open(path, registry: wide)) + { + writer.Commit(batch => + { + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); + batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] + { + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Wing), "stale wing") + }); + }); + } + + using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) + narrow.BackingStores[0].PushAll(); + + Assert.That(OverriddenSlotsOn(path, "drift-variant"), Is.EqualTo(new[] { 0 }), "the dropped slot's override is gone from the flushed record"); + + using var reused = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftReused) })); + Assert.That(reused.Get(new CultRecordKey("drift-base"))!.Colour, Is.Empty); + Assert.That(reused.Get(new CultRecordKey("drift-variant"))!.Colour, Is.Empty, "the variant inherits its base's colour, not the old wing"); + } + + [Test] + public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnLoadAndShedOnTheNextFlush() + { + var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", + CultVariantOverride.Set(99, new byte[] { 0x01 }), + TextOverride(0, "hand variant"), + TextOverride(2, "hand-code"))); + using (var cache = Open(path)) + { + var variantReport = cache.BackingStores[0].LastSchemaMigrationReports.Single(report => report.IgnoredExtraSlots.Length > 0); + Assert.That(variantReport.IgnoredExtraSlots, Is.EqualTo(new[] { 99 })); + Assert.That(variantReport.Warnings.Select(warning => warning.Code), Has.Some.EqualTo("ignored_extra_slot")); + Assert.That(variantReport.Warnings.Select(warning => warning.Message), Has.Some.Contain("hand-variant")); + cache.BackingStores[0].PushAll(); + } + + Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 })); + using var reopened = Open(path); + Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.Empty); + Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + } + + private static int[] OverriddenSlotsOn(string path, string key) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records + .Single(record => record.Key == key).Variant!.Overrides.Select(entry => entry.Path[0].Slot).ToArray(); + // ---- flatten resolves against the same batch ---- [Test] @@ -950,6 +1084,22 @@ namespace GameCult.Caching.Tests public string Wing = string.Empty; } + // A later type that reuses the slot Narrow dropped, for another member. + [CultDocument("tests.variant_drift", "tests.variant_drift.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class DriftReused + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public int Power; + + [Key(2)] + public string Colour = string.Empty; + } + [CultDocument("tests.variant_drift", "tests.variant_drift.v2")] [MessagePackObject(AllowPrivate = true)] internal sealed class DriftNarrow From b572823d0d9ed7744d59f0cef03ccb647d3e7779 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:28:24 +0200 Subject: [PATCH 11/16] Refuse variants after the selection is applied, and never tell a subscriber about the removal of one it was only refused The selection paths (SelectPage, SelectAll) refused every stored variant before the selection ran, so once any variant existed a hop-bearing subscription threw out of its change handler and went silent. They now refuse only a variant the selection selects; the subscription server delivers that as a typed error and keeps its projection. The database server no longer sends removed for a key the subscriber only ever received as an error. A flatten racing the delivery check is recorded in a comment. Co-Authored-By: Claude Sonnet 5 --- .../CultNetDatabaseServer.cs | 52 ++++-- .../CultNetDatabaseSubscriptionServer.cs | 24 ++- .../CultNetDocumentRegistry.cs | 28 ++- .../CultNetDocumentRegistryVariantTests.cs | 160 ++++++++++++++++++ 4 files changed, 240 insertions(+), 24 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 0dc489ed..d53315ed 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Threading.Tasks; @@ -502,27 +503,48 @@ namespace GameCult.Networking string subscriptionId, CultNetServerPeer peer) { + // Keys this subscriber only ever received as a refusal: it was never given them as records. + var refused = new HashSet(StringComparer.Ordinal); return _database.WatchAllChanges().Subscribe(change => { - CultNetDatabaseChangeRawMessage? outbound; - try - { - outbound = CreateChangeMessage(change, subscriptionId, selection); - } - catch (NotSupportedException refusal) - { - // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. - peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); - return; - } - + var outbound = DeliverChange(change, subscriptionId, selection, refused, error => peer.SendCultNet(error)); if (outbound != null) - { peer.SendCultNet(outbound); - } }); } + // One subscriber's view of one change. A variant is not a record CultNet can carry (Q6): the subscriber is told, by key, and + // is not later told the variant was removed, because it never held it. + internal CultNetDatabaseChangeRawMessage? DeliverChange( + object change, + string subscriptionId, + CultNetSelection selection, + HashSet refused, + Action sendError) + { + CultNetDatabaseChangeRawMessage? outbound; + try + { + outbound = CreateChangeMessage(change, subscriptionId, selection); + } + catch (NotSupportedException refusal) + { + var refusedKey = change.GetType().GetProperty("Key")?.GetValue(change) is CultRecordKey changed ? changed.Value : string.Empty; + lock (refused) + refused.Add(refusedKey); + sendError(new CultNetErrorMessage { Error = refusal.Message }); + return null; + } + + if (outbound == null) + return null; + var recordKey = outbound.RecordKey ?? outbound.Document?.RecordKey ?? string.Empty; + bool wasRefused; + lock (refused) + wasRefused = refused.Remove(recordKey); + return wasRefused && outbound.ChangeKind == "removed" ? null : outbound; + } + internal CultNetDatabaseChangeRawMessage? CreateChangeMessage( object change, string subscriptionId, @@ -565,7 +587,7 @@ namespace GameCult.Networking { MessageId = Guid.NewGuid().ToString("N"), SubscriptionId = subscriptionId, - ChangeKind = kind ==CultNetDatabaseChangeKind.Added ? "added" : "updated", + ChangeKind = kind == CultNetDatabaseChangeKind.Added ? "added" : "updated", Document = _database.Documents.ToRawRecord(descriptor, key, document, DateTimeOffset.UtcNow.ToString("O")) }; } diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index 359a9dfb..bb1f3900 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -204,6 +204,13 @@ namespace GameCult.Networking peer.SendCultNet(CultNetErrorMessage.ForReferenceOutsideTarget(ex)); return Task.CompletedTask; } + catch (NotSupportedException refusal) + { + // Q6: a snapshot that selects a variant is refused to this subscriber as a typed error. + Withdraw(key, sendRemovals: false, forgetRequest: true); + peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + return Task.CompletedTask; + } catch { Withdraw(key, sendRemovals: false, forgetRequest: true); @@ -342,9 +349,20 @@ namespace GameCult.Networking if (!_projections.TryGetValue(key, out var projection)) return; var legacyRequest = request.ToLegacyShape(key.Id); var authorized = _authorizeRequest?.Invoke(legacyRequest, key.Peer) != false; - var next = authorized - ? CreateProjectedSnapshot(request, key.Peer) - : ProjectedSnapshot.Empty; + ProjectedSnapshot next; + try + { + next = authorized + ? CreateProjectedSnapshot(request, key.Peer) + : ProjectedSnapshot.Empty; + } + catch (NotSupportedException refusal) + { + // Q6: the selection selects a variant, which CultNet cannot carry. This subscriber is told, by key, and keeps what + // it was delivered; a change never throws out of the change handler. + key.Peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + return; + } // With _projectRecord gone (docs/cultnet-selection-cut.md, S2-2), a projection's dictionary // key is always the record's own RecordKey/SchemaId, so the two can never disagree here; diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 356761f5..5df34e98 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -391,11 +391,13 @@ namespace GameCult.Networking var stored = cache.AllStoredDocuments; if (rowFilter != null) stored = stored.Where(entry => rowFilter(entry.Descriptor, entry.Key)); - var rows = stored + var entries = stored.ToArray(); + var rows = entries .Select(entry => new CultNetSelectionEvaluator.Row( - entry.Descriptor, entry.Key, RefuseVariant(entry).Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) + entry.Descriptor, entry.Key, entry.Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); + RefuseSelectedVariants(entries, full.Ordered); // R-Q: a page's asOf is exact only when every matched row is committed through one shard's // log - a caller that knows about shards (CultNetDatabaseServer) supplies shardIdOf; one // that does not (the raw v0/shard-scoped rowFilter paths, which are already bounded to a @@ -485,12 +487,14 @@ namespace GameCult.Networking var stored = cache.AllStoredDocuments; if (rowFilter != null) stored = stored.Where(entry => rowFilter(entry.Descriptor, entry.Key)); - var rows = stored + var entries = stored.ToArray(); + var rows = entries .Select(entry => new CultNetSelectionEvaluator.Row( - entry.Descriptor, entry.Key, RefuseVariant(entry).Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) + entry.Descriptor, entry.Key, entry.Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); + RefuseSelectedVariants(entries, full.Ordered); var records = full.Ordered.Select(row => ToRawRecord(row, options)).ToArray(); var wantDocument = selection.Projection == CultNetSelectionProjections.Document; var edges = selection.HasHop @@ -550,12 +554,24 @@ namespace GameCult.Networking } // CultNet carries no variant deltas yet, and a resolved view sent as a plain record would be a lie about the store. - private static CultStoredDocument RefuseVariant(CultStoredDocument entry) => - entry.Variant == null ? entry : throw VariantRefusal(entry); + // The refusal follows the selection: a variant is refused only when the selection selects it, never for existing. + private static void RefuseSelectedVariants(IReadOnlyList entries, IReadOnlyList selected) + { + if (selected.Count == 0 || !entries.Any(entry => entry.Variant != null)) + return; + var variants = entries.Where(entry => entry.Variant != null).ToDictionary(entry => entry.Key.Value, StringComparer.Ordinal); + foreach (var row in selected) + { + if (variants.TryGetValue(row.Key.Value, out var variant)) + throw VariantRefusal(variant); + } + } // The live-change funnel's refusal: a load or write that lands a variant must not reach a subscriber as a record. internal static void RefuseVariantChange(CultCache cache, CultRecordKey key) { + // Recorded, not fixed: a Flatten committed on another thread between the change's admission and this read makes + // the record plain here, and its resolved view is delivered. The window is one thread hop wide. if (cache.GetStored(key) is { Variant: not null } entry) throw VariantRefusal(entry); } diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 8c70c31e..2ae6dfab 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -182,6 +182,153 @@ namespace GameCult.Networking.Tests Assert.That(refused.InnerException!.Message, Does.Contain(VariantKey.Value).And.Contain(BaseKey.Value)); } + + // A hop-bearing selection reconciles the whole store on every change. The refusal follows the selection: a subscriber + // whose selection never selects the variant keeps receiving; one that does is told, as a typed error, not a silent stop. + [Test] + public async Task AHopBearingSubscriptionReconcilesPastAVariantItDoesNotSelectAndIsToldWhenItDoes() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var wireSchemaId = registry.GetRequired().SchemaId; + cache.Commit(batch => + { + batch.Upsert(typeof(VariantWireFixture), new VariantWireFixture { Name = "base", Power = 1 }, BaseKey); + batch.Upsert(typeof(VariantCiterFixture), Citer("citer-one", BaseKey), new CultRecordKey("citer-one")); + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "variant-hop-server", Socket = socket }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("variant-hop-client"); + var snapshots = new ConcurrentQueue(); + var changes = new ConcurrentQueue(); + var errors = new ConcurrentQueue(); + client.OnCultNet(snapshots.Enqueue); + client.OnCultNet(changes.Enqueue); + client.OnCultNet(errors.Enqueue); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + await WaitUntilAsync(() => client.Connected); + string Diagnostic() => + $"changes={string.Join(",", changes.Select(c => c.SubscriptionId + ":" + c.ChangeKind + ":" + (c.Document?.RecordKey ?? c.RecordKey)))} " + + $"errors={string.Join(",", errors.Select(e => e.Error))}"; + + // "citers": rows citing the base. It never selects the variant. + client.SendCultNet(new CultNetDatabaseSubscribeV1Message + { + MessageId = "citers", + SubscriptionId = "citers", + IncludeSnapshot = true, + Selection = new CultNetSelection + { + Cites = new CultNetCitation { Target = new CultNetRecordRef { SchemaId = wireSchemaId, RecordKey = BaseKey.Value }, Role = "Ref" } + } + }); + await WaitUntilAsync(() => snapshots.Count == 1, Diagnostic); + + cache.Commit(batch => batch.UpsertVariant(VariantKey, BaseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big") + })); + cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-two", BaseKey), new CultRecordKey("citer-two"))); + + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-two"), Diagnostic); + Assert.That(errors, Is.Empty, "a subscriber that never selects the variant is never told about it"); + + // "cited": wire rows some citer cites. Once a citer cites the variant, it selects it. + client.SendCultNet(new CultNetDatabaseSubscribeV1Message + { + MessageId = "cited", + SubscriptionId = "cited", + IncludeSnapshot = true, + Selection = new CultNetSelection + { + Schemas = new[] { wireSchemaId }, + Cited = new CultNetIncoming { Role = "Ref", Exists = true } + } + }); + await WaitUntilAsync(() => snapshots.Count == 2, Diagnostic); + Assert.That(errors, Is.Empty, "only the base is cited yet"); + cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-three", VariantKey), new CultRecordKey("citer-three"))); + + await WaitUntilAsync(() => errors.Any(error => error.Error.Contains(VariantKey.Value)), Diagnostic); + Assert.That(changes.Where(change => change.SubscriptionId == "cited").Select(change => change.Document?.RecordKey), + Does.Not.Contain(VariantKey.Value), "a variant never reaches a subscriber as a record"); + + // The first subscriber is still alive after the refusal. + cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-four", BaseKey), new CultRecordKey("citer-four"))); + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-four"), Diagnostic); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + + private static CultNetDatabaseChangeRawMessage? Deliver( + CultNetDatabaseServer server, + object change, + CultNetSelection selection, + System.Collections.Generic.HashSet refused, + System.Collections.Generic.List errors) + { + var method = typeof(CultNetDatabaseServer).GetMethod("DeliverChange", BindingFlags.Instance | BindingFlags.NonPublic)!; + return (CultNetDatabaseChangeRawMessage?)method.Invoke(server, new object[] + { + change, "sub", selection, refused, (Action)errors.Add + }); + } + + private static VariantCiterFixture Citer(string name, CultRecordKey target) => + new() { Name = name, Ref = new CultRecordRef(target) }; + + // A subscriber told a variant only as an error is not later told it was removed: it never held it. + [Test] + public async Task RemovingAVariantTheSubscriberWasOnlyToldAboutAsAnErrorSendsNoRemoval() + { + var (cache, documents) = await BuildFixtureAsync(withVariant: true); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var schemaId = cache.Registry.GetRequired().SchemaId; + var selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document }; + var refused = new System.Collections.Generic.HashSet(); + var errors = new System.Collections.Generic.List(); + + var added = new CultNetDatabaseChange( + CultNetDatabaseChangeKind.Added, VariantKey, schemaId, database.Shards[0], cache.Get(VariantKey)!, previousDocument: null); + Assert.That(Deliver(databaseServer, added, selection, refused, errors), Is.Null); + Assert.That(errors, Has.Count.EqualTo(1)); + + var removed = new CultNetDatabaseChange( + CultNetDatabaseChangeKind.Removed, VariantKey, schemaId, database.Shards[0], document: null, previousDocument: cache.Get(VariantKey)!); + Assert.That(Deliver(databaseServer, removed, selection, refused, errors), Is.Null, + "no removal for a key the subscriber never held as a record"); + + // A record it did hold is still told about. + var baseRemoved = new CultNetDatabaseChange( + CultNetDatabaseChangeKind.Removed, BaseKey, schemaId, database.Shards[0], document: null, previousDocument: cache.Get(BaseKey)!); + Assert.That(Deliver(databaseServer, baseRemoved, selection, refused, errors)?.ChangeKind, Is.EqualTo("removed")); + } + private static async Task WaitUntilAsync(Func condition, Func? diagnostic = null) { var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); @@ -205,5 +352,18 @@ namespace GameCult.Networking.Tests [Key(1)] public int Power; } + + [CultDocument("variants.wire-citer-fixture", "variants.wire-citer-fixture.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class VariantCiterFixture + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + [CultReference(typeof(VariantWireFixture))] + public CultRecordRef Ref; + } } } From 4f632fbf4616b6018af88cd15f02090a4b93df55 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:42:32 +0200 Subject: [PATCH 12/16] Drive the hop-subscription test through the database write path Batch commits on the cache do not reach the database's change stream, so the subscriber's citers were written with database.PutAsync; the variant goes in through UpsertVariantAsync. On the previous source the second subscriber stops receiving after the variant exists (red); it now keeps receiving. Co-Authored-By: Claude Sonnet 5 --- .../CultNetDocumentRegistryVariantTests.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 2ae6dfab..7aeadc15 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -243,13 +243,16 @@ namespace GameCult.Networking.Tests }); await WaitUntilAsync(() => snapshots.Count == 1, Diagnostic); - cache.Commit(batch => batch.UpsertVariant(VariantKey, BaseKey, new[] + await database.PutAsync(new CultRecordKey("citer-two"), Citer("citer-two", BaseKey)); + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-two"), Diagnostic); + + // With a variant in the store, the same subscriber keeps receiving. + await cache.UpsertVariantAsync(VariantKey, BaseKey, new[] { cache.Override(nameof(VariantWireFixture.Name), "big") - })); - cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-two", BaseKey), new CultRecordKey("citer-two"))); - - await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-two"), Diagnostic); + }); + await database.PutAsync(new CultRecordKey("citer-three"), Citer("citer-three", BaseKey)); + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-three"), Diagnostic); Assert.That(errors, Is.Empty, "a subscriber that never selects the variant is never told about it"); // "cited": wire rows some citer cites. Once a citer cites the variant, it selects it. @@ -266,15 +269,15 @@ namespace GameCult.Networking.Tests }); await WaitUntilAsync(() => snapshots.Count == 2, Diagnostic); Assert.That(errors, Is.Empty, "only the base is cited yet"); - cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-three", VariantKey), new CultRecordKey("citer-three"))); + await database.PutAsync(new CultRecordKey("citer-four"), Citer("citer-four", VariantKey)); await WaitUntilAsync(() => errors.Any(error => error.Error.Contains(VariantKey.Value)), Diagnostic); Assert.That(changes.Where(change => change.SubscriptionId == "cited").Select(change => change.Document?.RecordKey), Does.Not.Contain(VariantKey.Value), "a variant never reaches a subscriber as a record"); // The first subscriber is still alive after the refusal. - cache.Commit(batch => batch.Upsert(typeof(VariantCiterFixture), Citer("citer-four", BaseKey), new CultRecordKey("citer-four"))); - await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-four"), Diagnostic); + await database.PutAsync(new CultRecordKey("citer-five"), Citer("citer-five", BaseKey)); + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-five"), Diagnostic); } finally { From 282e337c2cf1f3ad0c5ef115df1c73009a5e5843 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 09:20:47 +0200 Subject: [PATCH 13/16] Variants C1: the held index tells the truth; a merge is judged on what the file will hold The held index kept one key per value, last writer winning, and RemoveKey dropped a value outright. Two consequences: removing or changing one of two plain holders left the other unfindable, and the variant index rule judged a variant against that lossy view, so a variant could land on a value a plain record still held and the store then refused to reopen. - _indexes maps a value to the keys holding it, oldest first. Index, Unindex and the removal maintain the set. GetByIndex finds the most recently indexed holder of a value, so removing the winner finds the other (behaviour change for plain duplicates: it used to find nothing). - One rule, RefuseVariantIndexSharing, serves both write orders. It reads the truthful index minus what the commit replaces or removes, plus what it lands. ResolveOne no longer judges indexes; ClaimIncomingIndexValues, HeldByVariants and RefusePlainSharingWithVariant are gone. - A conditional commit merged onto the file is judged by the cache's own resolution (CacheBackingStore.Judging) on the merged set: records other writers added, records they removed, and the batch. Another writer's variant sharing a value with the batch, or a base removal that strands another writer's variant, is refused with the named refusal instead of writing an unopenable store. - Shedding is one decision at load. ToStoredDocument resolves the schema once and returns the report with it; a variant's override of a slot the type lacks is dropped from the delta there, so flush needs no predicate and every store reports through the same surface (ReportVariantSheds and OverridesTheTypeHas are gone). The directory store refuses variants and loads through the same call. Co-Authored-By: Claude Sonnet 5 --- .../DirectoryMessagePackBackingStore.cs | 4 +- src/GameCult.Caching/CultCache.cs | 272 +++++++++--------- tests/GameCult.Caching.Tests/VariantTests.cs | 197 +++++++++++++ 3 files changed, 329 insertions(+), 144 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 3ef36be2..8244d76a 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -339,11 +339,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var started = tracePages ? Stopwatch.GetTimestamp() : 0L; var metadata = records[index]; var record = ReadPersistedRecordPage(metadata, out var pagePayload); - recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalogEntries); storedRecords[index] = ToStoredDocument( record, catalogEntries, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry)); + (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), + out recordReports[index]); if (tracePages) { pageBytes[index] = pagePayload.LongLength; diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2d4e7137..65c75b93 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1799,7 +1799,8 @@ namespace GameCult.Caching private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); private readonly Dictionary> _names = new(); - private readonly Dictionary<(Type Type, string Alias), Dictionary> _indexes = new(); + // Every key holding a value, oldest first: the index is truthful about duplicates, so removing one holder never hides another. + private readonly Dictionary<(Type Type, string Alias), Dictionary>> _indexes = new(); private readonly Dictionary _globals = new(); private readonly CultCodec? _codec; // Keys holding a variant: derived from the entries by Apply, never decided anywhere else. @@ -1934,6 +1935,7 @@ namespace GameCult.Caching store.AttachRegistry(_registry); store.Cache = this; store.Loaded = (loaded, dropped) => Admit(loaded, dropped, store, (_, _) => CultCommitOutcome.Committed); + store.Judging = (arriving, departing) => { Resolve(arriving, departing, store); }; _stores.Add(candidate); try { @@ -1943,6 +1945,7 @@ namespace GameCult.Caching { _stores.Remove(candidate); store.Loaded = null; + store.Judging = null; store.Cache = null; throw; } @@ -2121,11 +2124,13 @@ namespace GameCult.Caching .SelectMany(pair => pair.Value.TryGetValue(name, out var key) ? new[] { key } : Array.Empty())); } + // A value held by one record finds it. Records of one type sharing a value (plain duplicates) find the most recently + // indexed holder, and removing that holder finds the one before it: a held value never finds nothing. public T? GetByIndex(string alias, string value) where T : class { return Single($"index {alias}='{value}'", () => _indexes .Where(pair => pair.Key.Alias == alias && typeof(T).IsAssignableFrom(pair.Key.Type)) - .SelectMany(pair => pair.Value.TryGetValue(value, out var key) ? new[] { key } : Array.Empty())); + .SelectMany(pair => pair.Value.TryGetValue(value, out var keys) ? new[] { keys[keys.Count - 1] } : Array.Empty())); } public void Dispose() @@ -2522,21 +2527,20 @@ namespace GameCult.Caching if (toResolve.Count == 0) { var unchanged = new VariantPlan(admitted); - RefusePlainSharingWithVariant(unchanged, incoming, gone); + RefuseVariantIndexSharing(unchanged, incoming, gone); return unchanged; } var pending = new HashSet(toResolve, StringComparer.Ordinal); var done = new Dictionary(StringComparer.Ordinal); var stack = new List(); - Dictionary<(Type Type, string Alias), Dictionary>? claimed = null; foreach (var key in toResolve) ResolveOne(key); var plan = new VariantPlan(admitted.Select(stored => done.TryGetValue(stored.Key.Value, out var resolved) ? resolved : stored).ToList()); foreach (var key in toResolve.Where(key => !incoming.ContainsKey(key))) plan.Dependents.Add(done[key]); - RefusePlainSharingWithVariant(plan, incoming, gone); + RefuseVariantIndexSharing(plan, incoming, gone); return plan; CultStoredDocument ResolveOne(string key) @@ -2567,9 +2571,6 @@ namespace GameCult.Caching var type = descriptor.DocumentType; var kept = new List>(); var seen = new HashSet(); - // Bad writes are refused; drift is for type changes after the fact. A load, a re-resolution and a flatten - // carry what was already stored, so they follow the plain-record outcome instead. - var writing = source == null && isVariant && incoming.ContainsKey(key); foreach (var entry in delta.Overrides) { if (entry.Op != CultOverrideOp.Set || entry.Path.Count != 1 || entry.Path[0].ElementId.Length != 0 || entry.Id.Length != 0) @@ -2579,14 +2580,10 @@ namespace GameCult.Caching if (!seen.Add(slot)) throw new InvalidOperationException($"Variant {key} overrides slot {slot} more than once; a variant has one override per slot."); var member = descriptor.RichMembers.FirstOrDefault(candidate => candidate.Slot == slot); + // Drift is the load's business (ToStoredDocument sheds an override of a slot the type lacks), so a delta + // reaching here names only slots the type has, unless someone is writing one it never had. if (member == null) - { - if (writing) - throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have."); - // A slot the type dropped soft-drifts exactly as the same member does on a plain record: ignored, - // reported by the store's schema resolution (LastSchemaMigrationReports), the variant inherits the base. - continue; - } + throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have."); try { @@ -2620,37 +2617,6 @@ namespace GameCult.Caching $"Variant {key} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); } } - - // The same rule for an indexed value: a unique index maps one value to one key, so a variant that - // inherited its base's value (or any other record's) would be found in place of it or lose it. - foreach (var pair in descriptor.IndexAccessors) - { - var value = pair.Value(document); - if (string.IsNullOrWhiteSpace(value)) - continue; - claimed ??= ClaimIncomingIndexValues(); - var claim = (type, pair.Key); - string? other = null; - if (claimed.TryGetValue(claim, out var claims) && claims.TryGetValue(value, out var claimant) && claimant != key) - other = claimant; - else if (_indexes.TryGetValue(claim, out var held) && held.TryGetValue(value, out var holder) && holder != key && - !incoming.ContainsKey(holder) && !gone.Contains(holder) && !pending.Contains(holder)) - other = holder; - if (other != null) - throw new InvalidOperationException( - $"Variant {key} and {other} both hold index {pair.Key} '{value}'; a variant must override its indexed member."); - } - } - - if (descriptor.IndexAccessors.Count > 0) - { - claimed ??= ClaimIncomingIndexValues(); - foreach (var pair in descriptor.IndexAccessors) - { - var value = pair.Value(document); - if (!string.IsNullOrWhiteSpace(value)) - MapOf(claimed, (type, pair.Key))[value] = key; - } } stack.RemoveAt(stack.Count - 1); @@ -2658,80 +2624,56 @@ namespace GameCult.Caching ? variant.Resolved(document) : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { Flattens = true }; } - - // The indexed values the plain records of this admission will hold, by (type, index) then value. - Dictionary<(Type Type, string Alias), Dictionary> ClaimIncomingIndexValues() - { - var claims = new Dictionary<(Type Type, string Alias), Dictionary>(); - foreach (var stored in incoming.Values.Where(stored => stored.Variant == null && !stored.Flattens)) - { - foreach (var pair in stored.Descriptor.IndexAccessors) - { - var value = pair.Value(stored.Document); - if (!string.IsNullOrWhiteSpace(value)) - MapOf(claims, (stored.Descriptor.DocumentType, pair.Key))[value] = stored.Key.Value; - } - } - - return claims; - } } - // The index rule is about the commit's result, so it holds for every commit that changes an indexed value, in either - // order: a variant resolving onto another record's value is refused in ResolveOne; a plain record landing on a - // variant's value is refused here, against the variants of the post-commit set (never the held index, which the plain - // write would already have overwritten). Plain-vs-plain sharing is not this rule's business. - private void RefusePlainSharingWithVariant( + // The one index rule, judged on the commit's result whatever the write order: a variant may not share an indexed + // value with another record. It reads the truthful held index (every holder of a value), minus the records this + // commit replaces or removes, plus the records it lands. Plain-vs-plain sharing is not this rule's business. + // A unique index maps a value to one key at lookup; a variant sharing it would be found in place of its + // holder or lose the value, and the store would refuse to reopen. + private void RefuseVariantIndexSharing( VariantPlan plan, IReadOnlyDictionary incoming, HashSet gone) { - Dictionary<(Type Type, string Alias), Dictionary>? variantValues = null; - foreach (var stored in plan.Admitted) + var landing = plan.Admitted.Concat(plan.Dependents).ToList(); + if (landing.All(stored => stored.Descriptor.IndexAccessors.Count == 0)) + return; + var replaced = new HashSet(incoming.Keys.Concat(gone).Concat(plan.Dependents.Select(stored => stored.Key.Value)), StringComparer.Ordinal); + var landingByKey = landing.ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); + + bool IsVariant(string key) => + landingByKey.TryGetValue(key, out var landed) ? landed.Variant != null : _variantKeys.Contains(key); + + foreach (var stored in landing) { - if (stored.Variant != null || stored.Descriptor.IndexAccessors.Count == 0) - continue; foreach (var pair in stored.Descriptor.IndexAccessors) { var value = pair.Value(stored.Document); if (string.IsNullOrWhiteSpace(value)) continue; - variantValues ??= HeldByVariants(plan, incoming, gone); - if (variantValues.TryGetValue((stored.Descriptor.DocumentType, pair.Key), out var values) && - values.TryGetValue(value, out var variantKey) && variantKey != stored.Key.Value) + var type = stored.Descriptor.DocumentType; + var holders = (_indexes.TryGetValue((type, pair.Key), out var held) && held.TryGetValue(value, out var heldKeys) + ? heldKeys.Where(heldKey => !replaced.Contains(heldKey)) + : Enumerable.Empty()) + .Concat(landing + .Where(other => other.Descriptor.DocumentType == type && pair.Value(other.Document) == value) + .Select(other => other.Key.Value)) + .Where(holder => holder != stored.Key.Value) + .Distinct(StringComparer.Ordinal) + .ToArray(); + if (stored.Variant != null && holders.Length > 0) throw new InvalidOperationException( - $"Record {stored.Key.Value} and variant {variantKey} both hold index {pair.Key} '{value}'; " + + $"Variant {stored.Key.Value} and {holders[0]} both hold index {pair.Key} '{value}'; a variant must override its indexed member."); + var variantHolder = holders.FirstOrDefault(IsVariant); + if (stored.Variant == null && variantHolder != null) + throw new InvalidOperationException( + $"Record {stored.Key.Value} and variant {variantHolder} both hold index {pair.Key} '{value}'; " + "a variant may not share an indexed value with another record."); } } } - // The indexed values the variants of the post-commit set hold: those this commit resolved, and those it left alone. - private Dictionary<(Type Type, string Alias), Dictionary> HeldByVariants( - VariantPlan plan, - IReadOnlyDictionary incoming, - HashSet gone) - { - var variants = new Dictionary(StringComparer.Ordinal); - foreach (var variantKey in _variantKeys.Where(variantKey => !incoming.ContainsKey(variantKey) && !gone.Contains(variantKey))) - variants[variantKey] = _entries[variantKey]; - foreach (var stored in plan.Admitted.Concat(plan.Dependents).Where(stored => stored.Variant != null)) - variants[stored.Key.Value] = stored; - - var held = new Dictionary<(Type Type, string Alias), Dictionary>(); - foreach (var stored in variants.Values) - { - foreach (var pair in stored.Descriptor.IndexAccessors) - { - var value = pair.Value(stored.Document); - if (!string.IsNullOrWhiteSpace(value)) - MapOf(held, (stored.Descriptor.DocumentType, pair.Key))[value] = stored.Key.Value; - } - } - - return held; - } - private static InvalidOperationException CycleRefusal(IEnumerable keys) => new($"Variant bases form a cycle: {string.Join(" -> ", keys)}."); @@ -2837,7 +2779,7 @@ namespace GameCult.Caching { var value = pair.Value(stored.Document); if (!string.IsNullOrWhiteSpace(value)) - MapOf(_indexes, (type, pair.Key))[value] = stored.Key.Value; + AddHolder(_indexes, (type, pair.Key), value, stored.Key.Value); } } @@ -2850,7 +2792,27 @@ namespace GameCult.Caching if (_names.TryGetValue(type, out var names)) RemoveKey(names, stored.Key.Value); foreach (var pair in _indexes.Where(pair => pair.Key.Type == type)) - RemoveKey(pair.Value, stored.Key.Value); + RemoveHolder(pair.Value, stored.Key.Value); + } + + // Re-indexing a key makes it the most recent holder of its value. + private static void AddHolder(Dictionary<(Type Type, string Alias), Dictionary>> indexes, (Type, string) index, string value, string key) + { + if (!indexes.TryGetValue(index, out var map)) + indexes[index] = map = new Dictionary>(StringComparer.Ordinal); + if (!map.TryGetValue(value, out var keys)) + map[value] = keys = new List(); + keys.Remove(key); + keys.Add(key); + } + + private static void RemoveHolder(Dictionary> map, string key) + { + foreach (var pair in map.ToArray()) + { + if (pair.Value.Remove(key) && pair.Value.Count == 0) + map.Remove(pair.Key); + } } private static Dictionary MapOf(Dictionary> maps, TKey key) @@ -2924,6 +2886,11 @@ namespace GameCult.Caching // any of it; if the cache refuses a record the call throws and the store keeps its previous view. protected internal Action, IReadOnlyList>? Loaded; + // Set by the cache at attach: the cache's own resolution (variants, their indexed values, their bases) run over a + // set of records a store is about to hold, mutating nothing. It throws the refusal the cache would give the same + // set arriving as a load. + protected internal Action, IReadOnlyList>? Judging; + private readonly object _detachedGate = new(); internal CultCache? Cache; @@ -2977,55 +2944,51 @@ namespace GameCult.Caching SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt, Payload = entry.Variant != null ? Array.Empty() : serializePayload(entry.Document), - Variant = entry.Variant == null ? null : new CultVariantDelta(entry.Variant.BaseKey, OverridesTheTypeHas(entry).ToArray()) + Variant = entry.Variant }; } - // A plain record sheds a member its type dropped when it is flushed; a variant sheds the override of that slot, so - // a later type reusing the slot number cannot resurrect it. The load's report names the shed slots. - private static IEnumerable OverridesTheTypeHas(CultStoredDocument entry) => - entry.Variant!.Overrides.Where(entryOverride => - entryOverride.Path.Count != 1 || entry.Descriptor.RichMembers.Any(member => member.Slot == entryOverride.Path[0].Slot)); - - // A variant's override of a slot its type lacks is ignored like an extra slot on a plain record. Schema resolution - // reports slots the persisted schema had and the type dropped; an override the schema never had is added here. - protected static CultSchemaMigrationReport ReportVariantSheds(CultSchemaMigrationReport report, CultStoredDocument stored) - { - if (stored.Variant == null) - return report; - var shed = stored.Variant.Overrides - .Where(entryOverride => entryOverride.Path.Count == 1 && - stored.Descriptor.RichMembers.All(member => member.Slot != entryOverride.Path[0].Slot)) - .Select(entryOverride => entryOverride.Path[0].Slot) - .Except(report.IgnoredExtraSlots) - .Distinct() - .OrderBy(slot => slot) - .ToArray(); - if (shed.Length == 0) - return report; - report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(shed).OrderBy(slot => slot).ToArray(); - report.Warnings = report.Warnings.Concat(shed.Select(slot => new CultSchemaMigrationWarning - { - Code = "ignored_extra_slot", - Message = $"Variant {stored.Key.Value} overrides slot {slot}, which {stored.Descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush." - })).ToArray(); - return report; - } - + // The one load surface: a persisted record resolved against the current type, with the report of what that + // resolution shed. A plain record sheds a member its type dropped by never carrying it into the document; a variant + // sheds the override of such a slot by never carrying it into the delta, so a later type reusing the slot number + // cannot resurrect it and a flush needs no rule of its own. The report is the schema diff's ignored slots plus the + // overrides whose slot the type does not have, including one the persisted schema never listed. protected CultStoredDocument ToStoredDocument( CultPersistedRecord record, IReadOnlyCollection catalog, - Func deserializePayload) + Func deserializePayload, + out CultSchemaMigrationReport report) { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); + report = resolution.Report; if (record.Variant != null) { + var descriptor = resolution.Descriptor; + bool Has(CultVariantOverride entryOverride) => + entryOverride.Path.Count != 1 || descriptor.RichMembers.Any(member => member.Slot == entryOverride.Path[0].Slot); + var shed = record.Variant.Overrides + .Where(entryOverride => !Has(entryOverride)) + .Select(entryOverride => entryOverride.Path[0].Slot) + .Distinct() + .OrderBy(slot => slot) + .ToArray(); + if (shed.Length > 0) + { + var alreadyReported = report.IgnoredExtraSlots; + report.IgnoredExtraSlots = alreadyReported.Concat(shed.Except(alreadyReported)).OrderBy(slot => slot).ToArray(); + report.Warnings = report.Warnings.Concat(shed.Select(slot => new CultSchemaMigrationWarning + { + Code = "ignored_extra_slot", + Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush." + })).ToArray(); + } + // The cache resolves it against its base when it admits the load. return new CultStoredDocument( new CultRecordKey(record.Key), record.StoredAt, - resolution.Descriptor, - new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides), + descriptor, + new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides.Where(Has).ToArray()), null); } @@ -3084,9 +3047,8 @@ namespace GameCult.Caching var persisted = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { - var report = Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog); - var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload); - reports.Add(ReportVariantSheds(report, stored)); + var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload, out var report); + reports.Add(report); persisted[stored.Key.Value] = stored; } @@ -3169,6 +3131,11 @@ namespace GameCult.Caching var catalog = ontoDisk ? disk.SchemaCatalog : Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()); + // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the + // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), + // what they removed (departing), and this batch. + if (ontoDisk && Judging != null) + JudgeMerge(request, disk); foreach (var entry in request.Deletes) records.Remove(entry.Key.Value); foreach (var entry in request.Upserts) @@ -3183,6 +3150,27 @@ namespace GameCult.Caching return CultCommitOutcome.Committed; } + private void JudgeMerge(CultCommitRequest request, CultPersistedStoreSnapshot disk) + { + var landing = request.Upserts.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); + var removing = request.Deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); + var onDisk = disk.Records.Select(record => record.Key).ToHashSet(StringComparer.Ordinal); + var arriving = new List(request.Upserts); + foreach (var record in disk.Records) + { + if (landing.Contains(record.Key) || removing.Contains(record.Key)) + continue; + if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == record.SchemaId) + continue; + arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _)); + } + + var departing = new List(request.Deletes); + departing.AddRange(Entries.Values.Where(known => + !onDisk.Contains(known.Key.Value) && !landing.Contains(known.Key.Value) && !removing.Contains(known.Key.Value))); + Judging!(arriving, departing); + } + private CultPersistedStoreSnapshot? ReadSnapshot() { FileInfo.Refresh(); diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index c81e892b..6f6654e2 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -749,6 +749,203 @@ namespace GameCult.Caching.Tests Assert.That(reloaded.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); } + // ---- the held index is truthful: every holder of a value, so no rule judges a variant against a lossy view ---- + + private static VariantGear Named(string name, string code) => new() { Name = name, Power = 1, Code = code }; + + private static CultRecordKey[] SharedCodePair(CultCache cache) + { + var p1 = new CultRecordKey("p1"); + var p2 = new CultRecordKey("p2"); + cache.Commit(batch => + { + batch.Upsert(typeof(VariantGear), Named("p1", "x"), p1); + batch.Upsert(typeof(VariantGear), Named("p2", "x"), p2); + }); + return new[] { p1, p2 }; + } + + private static void VariantOnCode(CultCache cache, string code) => + cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + cache.Override(nameof(VariantGear.Name), "laser big"), + cache.Override(nameof(VariantGear.Code), code) + })); + + [Test] + public void PlainDuplicatesFindTheMostRecentHolderAndRemovingEitherLeavesTheOtherFindable() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + var pair = SharedCodePair(cache); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "the last writer"); + + cache.Remove(pair[1]); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "removing the winner finds the other holder"); + + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p2", "x"), pair[1])); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2")); + cache.Remove(pair[0]); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "removing the loser changes nothing"); + cache.Remove(pair[1]); + Assert.That(cache.GetByIndex("code", "x"), Is.Null); + cache.BackingStores[0].PushAll(); + } + + using var reopened = Open(path); + Assert.That(reopened.GetByIndex("code", "x"), Is.Null); + } + + [Test] + public void AVariantOnAValueAPlainRecordStillHoldsAfterItsTwinWasRemovedIsRefusedAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + SeedBase(cache); + var pair = SharedCodePair(cache); + cache.Remove(pair[1]); + + var refused = Refused(() => VariantOnCode(cache, "x")); + Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain("p1").And.Contain("code").And.Contain("'x'")); + Assert.That(cache.Get(BigKey), Is.Null, "nothing landed"); + cache.BackingStores[0].PushAll(); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(BigKey), Is.Null); + Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); + } + + [Test] + public void AVariantOnAValueAPlainRecordStillHoldsAfterItsTwinChangedItsValueIsRefusedAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + SeedBase(cache); + var pair = SharedCodePair(cache); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p2", "y"), pair[1])); + + var refused = Refused(() => VariantOnCode(cache, "x")); + Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain("p1").And.Contain("'x'")); + Assert.DoesNotThrow(() => VariantOnCode(cache, "z"), "an unshared value is fine"); + cache.BackingStores[0].PushAll(); + } + + using var reopened = Open(path); + Assert.That(reopened.GetByIndex("code", "z")!.Name, Is.EqualTo("laser big")); + Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); + } + + // ---- a merge onto the file is judged on the set the file will hold ---- + + [Test] + public void AConditionalCommitThatWouldShareAnIndexedValueWithAnotherWritersVariantIsRefusedAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + using (var seed = Open(path)) + SeedBase(seed); + + using (var a = Open(path)) + using (var b = Open(path)) + { + a.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] + { + a.Override(nameof(VariantGear.Name), "laser big"), + a.Override(nameof(VariantGear.Code), "shared") + })); + + var refused = Refused(() => b.Commit(batch => + { + batch.Expect(other, null); + batch.Upsert(typeof(VariantGear), Named("other", "shared"), other); + })); + Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("shared")); + Assert.That(b.Get(other), Is.Null, "nothing landed in memory"); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(other), Is.Null, "nothing landed on the file"); + Assert.That(reopened.GetByIndex("code", "shared")!.Name, Is.EqualTo("laser big")); + } + + [Test] + public void AConditionalCommitThatRemovesAnotherWritersVariantsBaseIsRefusedAndTheStoreReopens() + { + var path = PathOf("gear.cc"); + using (var seed = Open(path)) + SeedBase(seed); + + using (var a = Open(path)) + using (var b = Open(path)) + { + var seen = b.Get(BaseKey)!; + SeedBig(a); + + var refused = Refused(() => b.Commit(batch => + { + batch.Expect(BaseKey, seen); + batch.Remove(BaseKey); + })); + Assert.That(refused.Message, Does.Contain(BaseKey.Value).And.Contain(BigKey.Value)); + Assert.That(b.Get(BaseKey), Is.Not.Null, "nothing landed in memory"); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(BaseKey), Is.Not.Null); + Assert.That(reopened.Get(BigKey)!.Name, Is.EqualTo("laser big")); + } + + [Test] + public void AConditionalCommitThatIsSoundOnTheMergedSetStillLands() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + using (var seed = Open(path)) + SeedBase(seed); + + using (var a = Open(path)) + using (var b = Open(path)) + { + SeedBig(a); + Assert.That(b.Commit(batch => + { + batch.Expect(other, null); + batch.Upsert(typeof(VariantGear), Named("other", "fine"), other); + }), Is.True); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(other), Is.Not.Null); + Assert.That(reopened.Get(BigKey), Is.Not.Null, "the other writer's variant survives the merge"); + } + + [Test] + public void AVariantLoadedUnderANarrowerTypeHoldsOnlyTheOverridesTheTypeHasAndReportsTheRest() + { + var path = PathOf("drift.cc"); + using (var writer = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }))) + { + writer.Commit(batch => + { + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); + batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] + { + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Wing), "stale wing") + }); + }); + } + + using var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) })); + Assert.That(narrow.GetStored(new CultRecordKey("drift-variant"))!.Variant!.Overrides.Select(entry => entry.Path[0].Slot), Is.EqualTo(new[] { 0 }), + "the dropped slot's override is shed when the record is loaded, not decided again at flush"); + Assert.That(narrow.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Does.Contain(2)); + } + // ---- the directory store refuses a variant page on load as it does on write ---- [Test] From a0baa542d0009d17dcdb8bb9a8033cb5f406e43a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 09:20:48 +0200 Subject: [PATCH 14/16] CultNet: refuse hop edges whose citer is a variant; withdraw a refused hop subscription An edge carries the reference payload of its citer's resolved document, and the citer is never a selected row under `cited`, so a variant citer slipped past the selected-row refusal. Edges are now checked by the same rule (typed NotSupportedException naming the variant), on the paged and the whole-set paths. Reconcile refused a hop subscriber whose selection selected a variant without withdrawing it, so it was told again on every store change. It withdraws as the subscribe path does. Co-Authored-By: Claude Sonnet 5 --- .../CultNetDatabaseSubscriptionServer.cs | 6 +- .../CultNetDocumentRegistry.cs | 24 +++-- .../CultNetDocumentRegistryVariantTests.cs | 96 ++++++++++++++++++- 3 files changed, 113 insertions(+), 13 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index bb1f3900..60bb66a5 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -358,8 +358,10 @@ namespace GameCult.Networking } catch (NotSupportedException refusal) { - // Q6: the selection selects a variant, which CultNet cannot carry. This subscriber is told, by key, and keeps what - // it was delivered; a change never throws out of the change handler. + // Q6: the selection selects a variant, which CultNet cannot carry. Refused as the subscribe path refuses it: the + // subscription is withdrawn, so the peer is told once, not once per store change; a change never throws out of + // the change handler. + Withdraw(key, sendRemovals: false, forgetRequest: true); key.Peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); return; } diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 5df34e98..eb23d503 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -397,7 +397,7 @@ namespace GameCult.Networking entry.Descriptor, entry.Key, entry.Document, ordinalOf(entry.Descriptor.SchemaId, entry.Key), entry.StoredAt)) .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); - RefuseSelectedVariants(entries, full.Ordered); + RefuseVariants(entries, full.Ordered.Select(row => row.Key)); // R-Q: a page's asOf is exact only when every matched row is committed through one shard's // log - a caller that knows about shards (CultNetDatabaseServer) supplies shardIdOf; one // that does not (the raw v0/shard-scoped rowFilter paths, which are already bounded to a @@ -417,6 +417,7 @@ namespace GameCult.Networking } var evaluation = CultNetSelectionEvaluator.Page(full, selection, asOf, cursorKey); + RefuseVariants(entries, evaluation.Edges.Select(edge => edge.From.Key)); var records = evaluation.Rows.Select(row => ToRawRecord(row, options)).ToArray(); var wantDocument = selection.Projection == CultNetSelectionProjections.Document; @@ -494,12 +495,13 @@ namespace GameCult.Networking .ToArray(); var full = CultNetSelectionEvaluator.EvaluateAll(_documents, rows, selection); - RefuseSelectedVariants(entries, full.Ordered); + RefuseVariants(entries, full.Ordered.Select(row => row.Key)); var records = full.Ordered.Select(row => ToRawRecord(row, options)).ToArray(); var wantDocument = selection.Projection == CultNetSelectionProjections.Document; - var edges = selection.HasHop - ? CultNetSelectionEvaluator.EdgesFor(full.Ordered, full).Select(edge => ToEdge(edge, wantDocument)).ToArray() - : null; + var hopEdges = selection.HasHop ? CultNetSelectionEvaluator.EdgesFor(full.Ordered, full) : null; + if (hopEdges != null) + RefuseVariants(entries, hopEdges.Select(edge => edge.From.Key)); + var edges = hopEdges?.Select(edge => ToEdge(edge, wantDocument)).ToArray(); return new CultNetSelectionPage { @@ -554,15 +556,17 @@ namespace GameCult.Networking } // CultNet carries no variant deltas yet, and a resolved view sent as a plain record would be a lie about the store. - // The refusal follows the selection: a variant is refused only when the selection selects it, never for existing. - private static void RefuseSelectedVariants(IReadOnlyList entries, IReadOnlyList selected) + // The refusal follows the selection: a variant is refused only when the selection selects it, never for existing. An + // edge carries its citer's reference payload, read from the citer's resolved document, so a variant citing is selected + // by the selection's hop exactly as a variant row is. + private static void RefuseVariants(IReadOnlyList entries, IEnumerable selected) { - if (selected.Count == 0 || !entries.Any(entry => entry.Variant != null)) + if (!entries.Any(entry => entry.Variant != null)) return; var variants = entries.Where(entry => entry.Variant != null).ToDictionary(entry => entry.Key.Value, StringComparer.Ordinal); - foreach (var row in selected) + foreach (var key in selected) { - if (variants.TryGetValue(row.Key.Value, out var variant)) + if (variants.TryGetValue(key.Value, out var variant)) throw VariantRefusal(variant); } } diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 7aeadc15..a4658117 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -275,9 +275,103 @@ namespace GameCult.Networking.Tests Assert.That(changes.Where(change => change.SubscriptionId == "cited").Select(change => change.Document?.RecordKey), Does.Not.Contain(VariantKey.Value), "a variant never reaches a subscriber as a record"); - // The first subscriber is still alive after the refusal. + // The first subscriber is still alive after the refusal; the refused one is withdrawn, not told again per change. + var told = errors.Count; await database.PutAsync(new CultRecordKey("citer-five"), Citer("citer-five", BaseKey)); await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "citers" && change.Document?.RecordKey == "citer-five"), Diagnostic); + await Task.Delay(200); + Assert.That(errors.Count, Is.EqualTo(told), "a refused subscription is withdrawn: one error, not one per store change"); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + + // An edge carries its citer's reference payload, read from the citer's resolved document. A selection whose hop + // reaches a variant citer is refused as a selection of a variant row is, though only the cited row is selected. + private static async Task<(CultCache Cache, CultNetDocumentRegistry Documents, CultNetSelection Selection, CultRecordKey CiterVariant)> BuildCitedByVariantFixtureAsync() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)) + .Register(CultNetDocumentBinding.ForDocument(registry)); + cache.Commit(batch => + { + batch.Upsert(typeof(VariantWireFixture), new VariantWireFixture { Name = "base", Power = 1 }, BaseKey); + batch.Upsert(typeof(VariantCiterFixture), Citer("citer-one", BaseKey), new CultRecordKey("citer-one")); + }); + var citerVariant = new CultRecordKey("citer-variant"); + await cache.UpsertVariantAsync(citerVariant, new CultRecordKey("citer-one"), new[] + { + cache.Override(nameof(VariantCiterFixture.Name), "citer variant") + }); + var selection = new CultNetSelection + { + Schemas = new[] { cache.Registry.GetRequired().SchemaId }, + Cited = new CultNetIncoming { Role = "Ref", Exists = true }, + Projection = CultNetSelectionProjections.Document + }; + return (cache, documents, selection, citerVariant); + } + + [Test] + public async Task AHopEdgeWhoseCiterIsAVariantIsRefusedNamingItAndAStoreWithoutOneAnswers() + { + var (cache, documents, selection, citerVariant) = await BuildCitedByVariantFixtureAsync(); + + var error = Assert.Throws(() => + documents.CreateSelectionResponse(cache, "m1", selection, ordinalOf: (_, _) => 1, asOf: 1))!; + Assert.That(error.Message, Does.Contain(citerVariant.Value)); + + cache.Commit(batch => batch.Remove(citerVariant)); + var answered = documents.CreateSelectionResponse(cache, "m2", selection, ordinalOf: (_, _) => 1, asOf: 1); + Assert.That(answered.Edges, Has.Length.EqualTo(1)); + Assert.That(answered.Edges![0].From.RecordKey, Is.EqualTo("citer-one")); + } + + [Test] + public async Task AHopSubscriptionWhoseEdgeCiterIsAVariantIsRefusedOnceAndWithdrawn() + { + var (cache, documents, selection, citerVariant) = await BuildCitedByVariantFixtureAsync(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "variant-edge-server", Socket = socket }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("variant-edge-client"); + var snapshots = new ConcurrentQueue(); + var errors = new ConcurrentQueue(); + client.OnCultNet(snapshots.Enqueue); + client.OnCultNet(errors.Enqueue); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + await WaitUntilAsync(() => client.Connected); + + client.SendCultNet(new CultNetDatabaseSubscribeV1Message + { + MessageId = "edges", + SubscriptionId = "edges", + IncludeSnapshot = true, + Selection = selection + }); + await WaitUntilAsync(() => errors.Any(error => error.Error.Contains(citerVariant.Value))); + Assert.That(snapshots, Is.Empty, "the snapshot's edges would carry the variant's resolved reference"); + Assert.That(errors, Has.Count.EqualTo(1)); } finally { From 2eec4368501f1739b3f5180d098de91a300aea41 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:12:20 +0200 Subject: [PATCH 15/16] Variants C1: the index rule is linear, a merge judges only when a variant is involved, duplicates are deterministic - RefuseVariantIndexSharing built its holder list per landing record by scanning every landing record, so opening a store with one variant was quadratic. It now groups what lands once and looks each (index, value) up once. Pinned by a count of index-accessor reads at two sizes, not a clock. - JudgeMerge decoded every unseen disk record before Resolve could exit early, so a plain-only store failed a conditional commit on another writer's schema it does not register. It now judges only when the batch, the disk (by the record's variant header, never its payload) or this cache holds a variant. - Plain duplicates find the holder with the smallest key (ordinal), so the choice is the same live, after a flush and after a reopen. The comment claiming "most recently indexed" was wrong across reload. - A dropped slot is reported once per record, not once by the schema diff and again by the variant. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 76 ++++++--- tests/GameCult.Caching.Tests/VariantTests.cs | 165 ++++++++++++++++++- 2 files changed, 210 insertions(+), 31 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 65c75b93..b2f06659 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1799,7 +1799,7 @@ namespace GameCult.Caching private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); private readonly Dictionary> _names = new(); - // Every key holding a value, oldest first: the index is truthful about duplicates, so removing one holder never hides another. + // Every key holding a value (unordered): the index is truthful about duplicates, so removing one holder never hides another. private readonly Dictionary<(Type Type, string Alias), Dictionary>> _indexes = new(); private readonly Dictionary _globals = new(); private readonly CultCodec? _codec; @@ -1823,6 +1823,9 @@ namespace GameCult.Caching public CultDocumentRegistry Registry => _registry; + // Read under the gate, by a store judging a merge. + internal bool HoldsVariants => _variantKeys.Count > 0; + public CultCodec? Codec => _codec; // The whole override of one member, encoded with this cache's codec, ready for UpsertVariant. @@ -2124,13 +2127,16 @@ namespace GameCult.Caching .SelectMany(pair => pair.Value.TryGetValue(name, out var key) ? new[] { key } : Array.Empty())); } - // A value held by one record finds it. Records of one type sharing a value (plain duplicates) find the most recently - // indexed holder, and removing that holder finds the one before it: a held value never finds nothing. + // A value held by one record finds it. Records of one type sharing a value (plain duplicates) find the holder with the + // smallest key (ordinal), whatever order they were written or loaded in: the choice is a function of the held set, so + // it is the same live, after a flush and after a reopen, and removing that holder finds the next. public T? GetByIndex(string alias, string value) where T : class { return Single($"index {alias}='{value}'", () => _indexes .Where(pair => pair.Key.Alias == alias && typeof(T).IsAssignableFrom(pair.Key.Type)) - .SelectMany(pair => pair.Value.TryGetValue(value, out var keys) ? new[] { keys[keys.Count - 1] } : Array.Empty())); + .SelectMany(pair => pair.Value.TryGetValue(value, out var keys) + ? new[] { keys.Aggregate((best, next) => string.CompareOrdinal(next, best) < 0 ? next : best) } + : Array.Empty())); } public void Dispose() @@ -2645,6 +2651,9 @@ namespace GameCult.Caching bool IsVariant(string key) => landingByKey.TryGetValue(key, out var landed) ? landed.Variant != null : _variantKeys.Contains(key); + // One pass over what lands, one lookup per (index, value): the rule is linear in the records landing. + var landedValues = new List<(CultStoredDocument Stored, (Type Type, string Alias) Index, string Value)>(); + var landed = new Dictionary<((Type Type, string Alias) Index, string Value), List>(); foreach (var stored in landing) { foreach (var pair in stored.Descriptor.IndexAccessors) @@ -2652,25 +2661,38 @@ namespace GameCult.Caching var value = pair.Value(stored.Document); if (string.IsNullOrWhiteSpace(value)) continue; - var type = stored.Descriptor.DocumentType; - var holders = (_indexes.TryGetValue((type, pair.Key), out var held) && held.TryGetValue(value, out var heldKeys) + var index = (stored.Descriptor.DocumentType, pair.Key); + landedValues.Add((stored, index, value)); + if (!landed.TryGetValue((index, value), out var landedKeys)) + landed[(index, value)] = landedKeys = new List(); + landedKeys.Add(stored.Key.Value); + } + } + + var holdersOf = new Dictionary<((Type Type, string Alias) Index, string Value), (string[] All, string[] Variants)>(); + foreach (var (stored, index, value) in landedValues) + { + if (!holdersOf.TryGetValue((index, value), out var known)) + { + var all = (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKeys) ? heldKeys.Where(heldKey => !replaced.Contains(heldKey)) : Enumerable.Empty()) - .Concat(landing - .Where(other => other.Descriptor.DocumentType == type && pair.Value(other.Document) == value) - .Select(other => other.Key.Value)) - .Where(holder => holder != stored.Key.Value) + .Concat(landed[(index, value)]) .Distinct(StringComparer.Ordinal) .ToArray(); - if (stored.Variant != null && holders.Length > 0) - throw new InvalidOperationException( - $"Variant {stored.Key.Value} and {holders[0]} both hold index {pair.Key} '{value}'; a variant must override its indexed member."); - var variantHolder = holders.FirstOrDefault(IsVariant); - if (stored.Variant == null && variantHolder != null) - throw new InvalidOperationException( - $"Record {stored.Key.Value} and variant {variantHolder} both hold index {pair.Key} '{value}'; " + - "a variant may not share an indexed value with another record."); + holdersOf[(index, value)] = known = (all, all.Where(IsVariant).ToArray()); } + + var self = stored.Key.Value; + var holder = known.All.FirstOrDefault(candidate => candidate != self); + if (stored.Variant != null && holder != null) + throw new InvalidOperationException( + $"Variant {self} and {holder} both hold index {index.Alias} '{value}'; a variant must override its indexed member."); + var variantHolder = known.Variants.FirstOrDefault(candidate => candidate != self); + if (stored.Variant == null && variantHolder != null) + throw new InvalidOperationException( + $"Record {self} and variant {variantHolder} both hold index {index.Alias} '{value}'; " + + "a variant may not share an indexed value with another record."); } } @@ -2795,7 +2817,6 @@ namespace GameCult.Caching RemoveHolder(pair.Value, stored.Key.Value); } - // Re-indexing a key makes it the most recent holder of its value. private static void AddHolder(Dictionary<(Type Type, string Alias), Dictionary>> indexes, (Type, string) index, string value, string key) { if (!indexes.TryGetValue(index, out var map)) @@ -2974,9 +2995,10 @@ namespace GameCult.Caching .ToArray(); if (shed.Length > 0) { - var alreadyReported = report.IgnoredExtraSlots; - report.IgnoredExtraSlots = alreadyReported.Concat(shed.Except(alreadyReported)).OrderBy(slot => slot).ToArray(); - report.Warnings = report.Warnings.Concat(shed.Select(slot => new CultSchemaMigrationWarning + // A slot the schema diff already reported is not reported twice. + var unreported = shed.Except(report.IgnoredExtraSlots).ToArray(); + report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(unreported).OrderBy(slot => slot).ToArray(); + report.Warnings = report.Warnings.Concat(unreported.Select(slot => new CultSchemaMigrationWarning { Code = "ignored_extra_slot", Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush." @@ -3134,7 +3156,7 @@ namespace GameCult.Caching // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), // what they removed (departing), and this batch. - if (ontoDisk && Judging != null) + if (ontoDisk && Judging != null && MergeInvolvesVariants(request, disk)) JudgeMerge(request, disk); foreach (var entry in request.Deletes) records.Remove(entry.Key.Value); @@ -3150,6 +3172,14 @@ namespace GameCult.Caching return CultCommitOutcome.Committed; } + // Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote + // (read from the record's header, never its payload), or one this cache holds. A plain-only store never decodes + // another writer's records, so a schema this cache does not register cannot fail a commit that involves no variant. + private bool MergeInvolvesVariants(CultCommitRequest request, CultPersistedStoreSnapshot disk) => + Cache?.HoldsVariants == true || + request.Upserts.Any(entry => entry.Variant != null) || + disk.Records.Any(record => record.Variant != null); + private void JudgeMerge(CultCommitRequest request, CultPersistedStoreSnapshot disk) { var landing = request.Upserts.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 6f6654e2..c8d7a8df 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -773,22 +773,22 @@ namespace GameCult.Caching.Tests })); [Test] - public void PlainDuplicatesFindTheMostRecentHolderAndRemovingEitherLeavesTheOtherFindable() + public void RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable() { var path = PathOf("gear.cc"); using (var cache = Open(path)) { var pair = SharedCodePair(cache); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "the last writer"); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "the smallest key"); - cache.Remove(pair[1]); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "removing the winner finds the other holder"); - - cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p2", "x"), pair[1])); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2")); cache.Remove(pair[0]); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "removing the loser changes nothing"); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "removing the holder finds the other one"); + + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p1", "x"), pair[0])); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); cache.Remove(pair[1]); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "removing the other changes nothing"); + cache.Remove(pair[0]); Assert.That(cache.GetByIndex("code", "x"), Is.Null); cache.BackingStores[0].PushAll(); } @@ -946,6 +946,155 @@ namespace GameCult.Caching.Tests Assert.That(narrow.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Does.Contain(2)); } + // ---- batch 4: linear rule, judge only when a variant is involved, deterministic duplicates, one warning ---- + + [CultDocument("tests.variant_counted", "tests.variant_counted.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class CountedGear + { + public static long CodeReads; + [IgnoreMember] + private string _code = string.Empty; + + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + [CultIndex("code")] + public string Code + { + get + { + System.Threading.Interlocked.Increment(ref CodeReads); + return _code; + } + set => _code = value; + } + } + + // The pin is a count of index-accessor reads, not a clock: linear work doubles with the records, quadratic work quadruples. + private long IndexReadsOpening(int records) + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(CountedGear) }); + var path = PathOf($"counted-{records}.cc"); + using (var writer = Open(path, registry: registry)) + { + writer.Commit(batch => + { + for (var index = 0; index < records; index++) + batch.Upsert(typeof(CountedGear), new CountedGear { Name = $"n{index}", Code = $"c{index}" }, new CultRecordKey($"k{index}")); + }); + writer.Commit(batch => batch.UpsertVariant(new CultRecordKey("kv"), new CultRecordKey("k0"), new[] + { + writer.Override(nameof(CountedGear.Name), "variant"), + writer.Override(nameof(CountedGear.Code), "variant-code") + })); + } + + System.Threading.Interlocked.Exchange(ref CountedGear.CodeReads, 0); + using (Open(path, registry: registry)) + return System.Threading.Interlocked.Read(ref CountedGear.CodeReads); + } + + [Test] + public void OpeningAStoreWithAVariantReadsEachIndexedValueAFixedNumberOfTimes() + { + var small = IndexReadsOpening(1000); + var large = IndexReadsOpening(2000); + Assert.That(large, Is.LessThan(small * 3), $"index reads: {small} at 1000 records, {large} at 2000; doubling the records must not quadruple the work"); + } + + [Test] + public void ASchemaThisCacheDoesNotRegisterDoesNotFailAConditionalCommitThatInvolvesNoVariant() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + var narrow = CultDocumentRegistry.ForTypes(new[] { typeof(VariantGear) }); + using (var seed = Open(path)) + SeedBase(seed); + + using (var a = Open(path)) + using (var b = Open(path, registry: narrow)) + { + a.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "foreign" }, new CultRecordKey("foreign"))); + Assert.That(b.Commit(batch => + { + batch.Expect(other, null); + batch.Upsert(typeof(VariantGear), Named("other", "fine"), other); + }), Is.True); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(other), Is.Not.Null); + Assert.That(reopened.Get(new CultRecordKey("foreign")), Is.Not.Null, "the other writer's record survives the merge"); + } + + [Test] + public void AVariantAnotherWriterLandedIsStillJudgedWhenThisCacheHoldsNoVariant() + { + var path = PathOf("gear.cc"); + var other = new CultRecordKey("other"); + using (var seed = Open(path)) + SeedBase(seed); + + using (var a = Open(path)) + using (var b = Open(path)) + { + VariantOnCode(a, "shared"); + Refused(() => b.Commit(batch => + { + batch.Expect(other, null); + batch.Upsert(typeof(VariantGear), Named("other", "shared"), other); + })); + } + } + + [Test] + public void PlainDuplicatesFindTheSmallestKeyLiveAfterAFlushAndAfterAReopen() + { + var path = PathOf("gear.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("nb", "x"), new CultRecordKey("b"))); + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("na", "x"), new CultRecordKey("a"))); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "live"); + cache.BackingStores[0].PushAll(); + Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "after a flush"); + } + + using (var reopened = Open(path)) + { + Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "after a reopen"); + reopened.Remove(new CultRecordKey("a")); + reopened.Commit(batch => batch.Upsert(typeof(VariantGear), Named("na", "x"), new CultRecordKey("a"))); + Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "rewriting the smaller key does not change the choice"); + } + } + + [Test] + public void ADroppedSlotIsReportedOncePerRecord() + { + var path = PathOf("drift.cc"); + using (var writer = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }))) + { + writer.Commit(batch => + { + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); + batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] + { + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Wing), "stale wing") + }); + }); + } + + using var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) })); + var variantReport = narrow.BackingStores[0].LastSchemaMigrationReports.Last(); + Assert.That(variantReport.Warnings.Count(warning => warning.Code == "ignored_extra_slot"), Is.EqualTo(1)); + Assert.That(variantReport.IgnoredExtraSlots, Is.EqualTo(new[] { 2 })); + } + // ---- the directory store refuses a variant page on load as it does on write ---- [Test] From cbd3b4f3ce3963979a7b30e88305c467e5c8a638 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:12:20 +0200 Subject: [PATCH 16/16] CultNet: a withdrawn hop subscription is refused with a typed error and its rows are withdrawn Reconcile's withdrawal sent an untyped error with no subscription id and left the peer holding the rows the subscription had delivered. It now sends removals for those rows, then a variant_unsupported error whose details carry field "subscriptionId" and the id (the existing details slots; no new wire key). The subscribe and per-change paths use the same code. The error schema and the Rust CultNetErrorCode accept variant_unsupported, so a Rust peer decodes it instead of refusing an unknown code. Co-Authored-By: Claude Sonnet 5 --- contracts/cultnet/cultnet.error.schema.json | 6 +++--- .../contracts/cultnet/cultnet.error.schema.json | 6 +++--- packages/cultnet-rs/src/contracts.rs | 5 ++++- .../CultNetDatabaseSubscriptionServer.cs | 13 +++++++------ src/GameCult.Networking/CultNetSchemaMessages.cs | 12 ++++++++++++ .../CultNetDocumentRegistryVariantTests.cs | 4 ++++ 6 files changed, 33 insertions(+), 13 deletions(-) diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index 8653cbd5..15880fa1 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index 8653cbd5..15880fa1 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index cc50171d..f31ade85 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -250,7 +250,7 @@ impl CultNetShardDescriptor { } } -/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the four strings +/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings /// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth /// string it does not know refuses to decode rather than guess a meaning for it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -260,6 +260,7 @@ pub enum CultNetErrorCode { CursorStale, CursorInvalid, ReferenceOutsideTarget, + VariantUnsupported, } impl CultNetErrorCode { @@ -269,6 +270,7 @@ impl CultNetErrorCode { Self::CursorStale => "cursor_stale", Self::CursorInvalid => "cursor_invalid", Self::ReferenceOutsideTarget => "reference_outside_target", + Self::VariantUnsupported => "variant_unsupported", } } @@ -278,6 +280,7 @@ impl CultNetErrorCode { "cursor_stale" => Ok(Self::CursorStale), "cursor_invalid" => Ok(Self::CursorInvalid), "reference_outside_target" => Ok(Self::ReferenceOutsideTarget), + "variant_unsupported" => Ok(Self::VariantUnsupported), other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")), } } diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index 60bb66a5..f3150740 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -208,7 +208,7 @@ namespace GameCult.Networking { // Q6: a snapshot that selects a variant is refused to this subscriber as a typed error. Withdraw(key, sendRemovals: false, forgetRequest: true); - peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + peer.SendCultNet(CultNetErrorMessage.ForVariantUnsupported(refusal.Message, subscriptionId)); return Task.CompletedTask; } catch @@ -330,7 +330,7 @@ namespace GameCult.Networking catch (NotSupportedException refusal) { // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. - peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + peer.SendCultNet(CultNetErrorMessage.ForVariantUnsupported(refusal.Message)); return; } @@ -358,11 +358,12 @@ namespace GameCult.Networking } catch (NotSupportedException refusal) { - // Q6: the selection selects a variant, which CultNet cannot carry. Refused as the subscribe path refuses it: the - // subscription is withdrawn, so the peer is told once, not once per store change; a change never throws out of + // Q6: the selection selects a variant, which CultNet cannot carry. The subscription is withdrawn, so the peer is + // told once, not once per store change; the rows it delivered are withdrawn too (removals first), so the peer + // does not keep them as live, and the typed error names the subscription that ended. A change never throws out of // the change handler. - Withdraw(key, sendRemovals: false, forgetRequest: true); - key.Peer.SendCultNet(new CultNetErrorMessage { Error = refusal.Message }); + Withdraw(key, sendRemovals: true, forgetRequest: true); + key.Peer.SendCultNet(CultNetErrorMessage.ForVariantUnsupported(refusal.Message, key.Id)); return; } diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index f343fad7..4daa76d5 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -546,6 +546,18 @@ namespace GameCult.Networking : null }; + /// + /// Builds the wire error for a selection that selects a document variant, which CultNet does not carry yet: code + /// variant_unsupported, details { field: "subscriptionId", value } when a subscription was ended by it. + /// + public static CultNetErrorMessage ForVariantUnsupported(string message, string? subscriptionId = null) => + new CultNetErrorMessage + { + Error = message, + Code = "variant_unsupported", + Details = subscriptionId == null ? null : new CultNetErrorDetails { Field = "subscriptionId", Value = subscriptionId } + }; + /// /// Builds the wire error for an out-of-target reference refusal (R-N): code /// reference_outside_target, no details. diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index a4658117..64de1093 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -272,6 +272,10 @@ namespace GameCult.Networking.Tests await database.PutAsync(new CultRecordKey("citer-four"), Citer("citer-four", VariantKey)); await WaitUntilAsync(() => errors.Any(error => error.Error.Contains(VariantKey.Value)), Diagnostic); + var refusal = errors.First(error => error.Error.Contains(VariantKey.Value)); + Assert.That(refusal.Code, Is.EqualTo("variant_unsupported")); + Assert.That(refusal.Details?.Value, Is.EqualTo("cited"), "the peer can tell which subscription ended"); + await WaitUntilAsync(() => changes.Any(change => change.SubscriptionId == "cited" && change.ChangeKind == "removed" && change.RecordKey == BaseKey.Value), Diagnostic); Assert.That(changes.Where(change => change.SubscriptionId == "cited").Select(change => change.Document?.RecordKey), Does.Not.Contain(VariantKey.Value), "a variant never reaches a subscriber as a record");