Slot tolerance: a record with unread slots is held read-only, not shed #124

Merged
libby merged 20 commits from hands/slot-tolerance into main 2026-11-12 02:45:08 +00:00
21 changed files with 1345 additions and 64 deletions

View file

@ -166,7 +166,7 @@ public static class CultDocumentMessagePackSerialization
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
{
// The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer
// The guard goes first and wraps whatever formatter the rest of the composite would hand out (a union, or a concrete int-keyed object for its slot count), so a consumer
// resolver still encodes an arm while the key check runs.
var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
@ -214,6 +214,19 @@ public static class CultDocumentMessagePackSerialization
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
}
// The same decode, with the slots it read past noted (CultSlotScope). Opened around the one decode and after the depth check,
// so a payload past the nesting limit is refused, never marked.
internal static CultDecoded DeserializeMarked(Type type, byte[] payload, CultDocumentRegistry registry)
{
if (registry == null) throw new ArgumentNullException(nameof(registry));
registry.GetRequired(type);
CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
using var scope = CultSlotScope.Open();
var document = MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
return new CultDecoded(document, scope.Finding());
}
/// <summary>
/// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member
/// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil.
@ -823,4 +836,7 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry);
}
internal override CultDecoded DeserializeMarked(Type documentType, byte[] payload) =>
CultDocumentMessagePackSerialization.DeserializeMarked(documentType, payload, Registry);
}

View file

@ -0,0 +1,47 @@
using System;
using GameCult.Caching;
namespace GameCult.Caching.MessagePack;
/// <summary>
/// The marks one payload decode noted: the guard's slot formatter reports every object whose array held more slots than its type
/// declares, with the nesting depth it sat at, and DeserializeMarked reads the finding out. Thread-bound and open only around the one
/// decode, so a decode outside it (a CultMesh read, a codec) notes nothing.
/// </summary>
internal sealed class CultSlotScope : IDisposable
{
[ThreadStatic] private static CultSlotScope? current;
private readonly CultSlotScope? previous;
private string? typeName;
private int slotCount;
private int declaredCount;
private int occurrences;
private bool allAtDepthZero = true;
private CultSlotScope() => previous = current;
internal static CultSlotScope? Current => current;
internal static CultSlotScope Open() => current = new CultSlotScope();
// 0 for the document's own object; each object nested inside it adds one (a union adds none, its arm is the object).
internal int Depth { get; set; }
internal void Note(string name, int slots, int declared)
{
if (occurrences++ == 0)
{
typeName = name;
slotCount = slots;
declaredCount = declared;
}
if (Depth > 0) allAtDepthZero = false;
}
internal CultUnreadSlots? Finding() =>
occurrences == 0 ? null : new CultUnreadSlots(typeName!, slotCount, declaredCount, occurrences, allAtDepthZero, explained: false);
public void Dispose() => current = previous;
}

View file

@ -1,14 +1,17 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using MessagePack;
using MessagePack.Formatters;
using MessagePack.Resolvers;
namespace GameCult.Caching.MessagePack;
/// <summary>
/// The one place an unknown union arm or an unnamed subtype is decided. A union is an abstract class or interface carrying
/// The one sensor of an unknown union arm, an unnamed subtype and a slot past the reader's declared slots. A union is an abstract class or interface carrying
/// its own [Union] attributes (never inherited: a union whose arms are also a union has its own key set). MessagePack-CSharp
/// still encodes the [key, armSlots] shape; this wraps its formatter so that a key the union does not declare is refused on
/// read, and a value whose runtime type is not a declared arm is refused on write, where MessagePack alone yields null and nil.
@ -18,11 +21,117 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol
public IMessagePackFormatter<T>? GetFormatter<T>()
{
var arms = typeof(T).GetCustomAttributes<UnionAttribute>(inherit: false).ToArray();
return arms.Length == 0 || inner.GetFormatter<T>() is not { } formatter ? null : new Formatter<T>(formatter, arms);
if (arms.Length == 0)
return inner.GetFormatter<T>() is { } objectFormatter && KeyLayout(objectFormatter) ? new SlotFormatter<T>(objectFormatter) : null;
return inner.GetFormatter<T>() is not { } formatter ? null : new Formatter<T>(formatter, arms);
}
private sealed class Formatter<T>(IMessagePackFormatter<T> inner, UnionAttribute[] arms) : IMessagePackFormatter<T>
// The formatter that writes and reads the bytes: the one a guard wrapper delegates to, else the formatter itself. A check that a
// generated formatter is the one in use asks this (tests/GameCult.Caching.GeneratorConsumer).
internal static object Inner(object formatter) => formatter is IWrapper wrapper ? wrapper.Inner : formatter;
private interface IWrapper
{
object Inner { get; }
}
// The one decision of whether a type is measured: the formatter the chain hands out for an int-keyed [MessagePackObject] type is
// MessagePack's own [Key]-layout formatter, the very instance StandardResolver (or its AllowPrivate twin) builds for it. A formatter a
// consumer resolver or a [MessagePackFormatter] attribute supplies owns its layout and its tolerance, and a [MessagePackObject]
// collection is written as its items by the generic resolver: none of those is measured. Identity, as ResolverClaim asks it.
private static bool KeyLayout<T>(IMessagePackFormatter<T> formatter)
{
if (typeof(T).IsAbstract || typeof(T).IsInterface ||
typeof(T).GetCustomAttribute<MessagePackObjectAttribute>(inherit: true) is not { KeyAsPropertyName: false })
return false;
if (ReferenceEquals(formatter, Of<T>(AttributeFormatterResolver.Instance)) || ReferenceEquals(formatter, Of<T>(DynamicGenericResolver.Instance)))
return false;
return ReferenceEquals(formatter, Of<T>(StandardResolver.Instance)) || ReferenceEquals(formatter, Of<T>(StandardResolverAllowPrivate.Instance));
}
private static IMessagePackFormatter<T>? Of<T>(IFormatterResolver resolver)
{
try
{
return resolver.GetFormatter<T>();
}
catch (Exception)
{
return null;
}
}
// Counts a concrete object's array header against its declared slots and notes the excess into the open CultSlotScope, then lets the
// inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through.
// Declared slots are what the type's own formatter writes: the array header of one instance written through it, measured on the
// first read inside a scope (not in GetFormatter, so a self-referential type never recurses into its own resolver cache) and kept.
private sealed class SlotFormatter<T>(IMessagePackFormatter<T> inner) : IMessagePackFormatter<T>, IWrapper
{
object IWrapper.Inner => inner;
private const int NotMeasured = -1;
// A type no instance of which can be written is not guarded: no count can exceed this.
private const int Unguarded = int.MaxValue;
private int declared = NotMeasured;
public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options) =>
inner.Serialize(ref writer, value, options);
public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
var scope = CultSlotScope.Current;
if (scope == null) return inner.Deserialize(ref reader, options);
var peek = reader.CreatePeekReader();
if (!peek.TryReadNil() && peek.NextMessagePackType == MessagePackType.Array)
{
var count = peek.ReadArrayHeader();
if (declared == NotMeasured) declared = Measure(options);
if (count > declared) scope.Note(typeof(T).Name, count, declared);
}
scope.Depth++;
try
{
return inner.Deserialize(ref reader, options);
}
finally
{
scope.Depth--;
}
}
// The uninitialized instance, else one from a parameterless constructor (public or not) when that write throws.
private int Measure(MessagePackSerializerOptions options)
{
if (Written(() => (T)RuntimeHelpers.GetUninitializedObject(typeof(T)), options) is { } slots) return slots;
var constructor = typeof(T).GetConstructor(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, null, Type.EmptyTypes, null);
return constructor != null && Written(() => (T)constructor.Invoke(null), options) is { } constructed ? constructed : Unguarded;
}
private int? Written(Func<T> instance, MessagePackSerializerOptions options)
{
try
{
var buffer = new ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
inner.Serialize(ref writer, instance(), options);
writer.Flush();
var written = new MessagePackReader(buffer.WrittenMemory);
return written.NextMessagePackType == MessagePackType.Array ? written.ReadArrayHeader() : null;
}
catch (Exception)
{
return null;
}
}
}
private sealed class Formatter<T>(IMessagePackFormatter<T> inner, UnionAttribute[] arms) : IMessagePackFormatter<T>, IWrapper
{
object IWrapper.Inner => inner;
private readonly HashSet<int> keys = arms.Select(arm => arm.Key).ToHashSet();
private readonly HashSet<Type> types = arms.Select(arm => arm.SubType).ToHashSet();

View file

@ -636,7 +636,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
record,
VersionOf(metadata.Payload),
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
(type, payload) => CultDocumentMessagePackSerialization.DeserializeMarked(type, payload, Registry),
out report);
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
stored.StoredSchemaId = metadata.SchemaId;

View file

@ -11,6 +11,7 @@
<PackageReference Include="System.Text.Json" Version="9.0.10" />
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
<InternalsVisibleTo Include="GameCult.Caching.GeneratorConsumer" />
<None Include="buildTransitive\GameCult.Caching.MessagePack.targets" Pack="true" PackagePath="buildTransitive\" />
</ItemGroup>
</Project>

View file

@ -202,6 +202,20 @@ flatten, remove, commit) is refused with the typed schema conflict before anythi
Declaring its id compatible on any registered type makes it writable and is the consent to the loss:
the next write stores it under the writing type's own id and sheds the slots.
A record whose payload carries a slot past the slots its reader declares, at any depth (nested object, list
element, union arm, unit arm), is read-only the same way, whatever id it is stored under. The one consent
is the declaration above, and it reaches only depth-0 slots the schema comparison reports ignored under a
listed compatible id, up to the highest slot it reports: a mark anywhere else, past that slot, or at depth 0
under the reader's own id, is not consented to. Declared slots are what the type's own formatter writes; only
a type MessagePack writes by its `[Key]` layout is checked, and one no instance of which can be written is not
(`cultlib-gaps:ruling:slot-unmeasurable-type`; see the schema-compatibility contract for both exemptions and
for the user code the measure runs).
A flatten of a variant whose base chain holds a record that is not owned is refused with that record's
refusal, because it would write the base's decoded view; the variant's own writes persist only its delta and
stay allowed. A plain write to another key of a value read from a read-only record is a copy this rule does
not police. `CultCache.IsWritable(key)` answers what the staging refusal answers for a removal or a write of
the record's type, read-only and missing home stores included; it does not predict that flatten.
The rule is checked again where the write lands, under the store's lock: a write or removal at a key
whose durable record is not owned, because another writer rewrote it after this cache
loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can
@ -535,15 +549,14 @@ Read directly when:
Allowed when:
- persisted schema has missing slots that the local schema can default
- persisted schema has extra slots the local reader can safely ignore or preserve
- slot meanings remain compatible
Soft migration should emit warnings, not silent shrugs.
A persisted schema with extra slots the local reader lacks is not soft: the record is read-only (see the ownership
rule above). Soft migration should emit warnings, not silent shrugs.
Typical warnings:
- missing slot defaulted
- unknown extra slot ignored
- nullable/default behavior widened
- field preserved but not surfaced natively

View file

@ -107,10 +107,48 @@ persisted slots honestly:
- persisted schema id differs, but the embedded catalog points to a compatible
local schema id
- persisted slot is missing locally and can be ignored
- persisted slot is missing locally and the record is read-only until its id is declared compatible
(see Extra slots are read-only, below)
- local slot is missing in persisted data and can fall back to the local
default value
Extra slots are read-only. A record whose payload carries a slot past the slots its reader declares, at any
depth (the record, a nested object, a list element, a union arm, a unit arm), is loaded read-only: it is
served, `CultStoredDocument.UnreadSlots` names the first type read with extra slots and both counts, and
`CultCache.IsWritable(key)` is false. A staged write or removal of it is refused with
`CultSchemaConflictException` (`UnreadSlots` set; the message names the type and the counts, never a slot
value). A missing trailing slot keeps its default and stays writable; a non-nil value in a retired gap below
the highest key is not extra. Declaring an id compatible consents only to depth-0 slots the schema comparison
reports ignored under that id, and only up to the highest slot it reports: a payload carrying a slot past that
one is not consented to. Any other mark (nested, list element, arm, unit arm, or depth 0 under the reader's
own id) keeps the record read-only whatever is declared.
A type's declared slots are the array its own formatter writes for one instance, never a reflected member
list, so a non-public `[Key]` member of an `AllowPrivate` type counts. Only a concrete int-keyed
`[MessagePackObject]` type whose formatter is MessagePack's own `[Key]`-layout formatter is checked. Not
checked, and so read past with any extra slots dropped on the next write, as before:
- a type formatted by anything else: its own `[MessagePackFormatter]`, a formatter of its own that a
`CultCacheFormatterResolver` (or any resolver ahead of MessagePack's) hands out, or the collection formatter
of a `[MessagePackObject]` collection type such as a `List<T>` subclass. Such a formatter owns its layout and
its tolerance. A resolver that hands out MessagePack's own `[Key]`-layout formatter (StandardResolver's, or
StandardResolverAllowPrivate's) does not exempt the type: it is checked and marks;
- a type no instance of which can be written: no write of an uninitialized instance and no write of one
from a parameterless constructor succeeds (for example a `[Key]` getter that throws on an uninitialized
instance and no parameterless constructor). This gap is ruled: `cultlib-gaps:ruling:slot-unmeasurable-type`.
Measuring runs user code once per type per process, on the first read of the type in a store load: the
type's `[Key]` getters on an uninitialized instance and, if that write throws, its parameterless
constructor (public or not) and the getters on that instance. Any failure of both leaves the type unchecked
for the rest of the process, and nothing else: it does not fail the read.
`Flatten` of a variant writes its base chain's decoded view into a plain record, so it is refused when any
record on that chain is not owned by its home store (a mark nothing explains, or undeclared drift that drops
slots), with that record's own refusal naming its key. Writing or removing the variant itself stays allowed:
it persists only its delta. `CultCache.IsWritable(key)` is the write's own answer for a removal or a write of
the record's type: it is also false when the record's home store is read-only or no attached store is home to
its type. The one write it does not predict is that flatten.
The cache emits a typed migration report with:
- exact vs compatible-drift classification

View file

@ -516,14 +516,22 @@ namespace GameCult.Caching
// resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it).
public string StoredSchemaId { get; internal set; }
// True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads
// it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots,
// True when the schema this record was loaded under left none of its persisted slots unread by Descriptor and its payload
// carried none past Descriptor's, so the type reads it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots,
// before any variant override is shed. A record this cache constructed or wrote is stored under Descriptor's own id, which
// the registry declares, so only a loaded record under another id is ever asked. The staging half asks it;
// ConditionsHold never reads it. Only the store's loaded view carries it: the cache's resolved copies and flattens are never
// asked.
internal bool StoredSchemaReadWhole { get; set; }
/// <summary>
/// What the store's decode of this record's payload found past the slots this build declares, at any depth; null when none. Set by
/// ToStoredDocument on the store's loaded view only (the cache's resolved copy of a variant does not carry it; ask
/// <see cref="CultCache.IsWritable"/>). A mark that is not <see cref="CultUnreadSlots.Explained"/> keeps the record
/// unowned even under a declared id: it cannot be written or removed (HeldOwned).
/// </summary>
public CultUnreadSlots? UnreadSlots { get; internal set; }
// The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for,
// so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record
// this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that
@ -2300,8 +2308,9 @@ namespace GameCult.Caching
// The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in
// memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records
// it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only,
// so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand.
// it rewrote; a second call finds none. A selected record that is not owned (neither declared nor read whole, or held
// read-only for slots past this build's, see CultStoredDocument.UnreadSlots) is read-only, so its home store's commit is
// refused with CultSchemaConflictException until a build that reads those slots rewrites it; commits to earlier stores stand.
public int MintElementIds()
{
var groups = Held(() => _entries.Values
@ -2656,6 +2665,22 @@ namespace GameCult.Caching
return _entries.TryGetValue(key.Value, out var stored) ? stored : null;
}
/// <summary>
/// Whether a removal of the key, or a write of the record's own type, would be let through: the write's own decision, asked of
/// the record the cache holds. False when no attached store is home to the record's type, when its home store is read-only, or
/// when that store holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's,
/// see <see cref="CultStoredDocument.UnreadSlots"/>). For a key the cache does not hold, the removal check every store makes of it
/// (false when a store lists it foreign). One write it does not predict: a Flatten of a variant whose base chain holds an unowned record is refused, though
/// the variant itself is writable. Live and advisory: the write still decides.
/// </summary>
public bool IsWritable(CultRecordKey key)
{
lock (Gate())
return _entries.TryGetValue(key.Value, out var held)
? WriteRefusal(held, upsert: false, null, out _) == null
: _stores.All(entry => entry.Store.Refusal(key.Value, null) == null);
}
public object? Get(CultRecordKey key)
{
lock (Gate())
@ -3151,6 +3176,8 @@ namespace GameCult.Caching
IReadOnlyCollection<string>? removed)
{
var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet<string>(StringComparer.Ordinal);
var batch = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
foreach (var stored in admitted) batch[stored.Key.Value] = stored;
var homes = new List<CacheBackingStore>();
var globals = new Dictionary<Type, string>(_globals);
var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal);
@ -3199,23 +3226,57 @@ namespace GameCult.Caching
void Claim(CultStoredDocument stored, bool upsert)
{
var home = Home(stored.Descriptor.DocumentType);
if (home == null)
if (WriteRefusal(stored, upsert, batch, out var home) is { } refusal) throw refusal;
if (home != null) homes.Add(home);
}
}
// The one answer to whether a record may be written (upsert: a write of its type) or its key removed, before anything is staged.
// Claim throws it; IsWritable asks it of the held record's removal form. Null when the write may land; home is where it lands,
// null for a store-less cache. batch holds the records the same batch admits, by key.
private Exception? WriteRefusal(CultStoredDocument stored, bool upsert, IReadOnlyDictionary<string, CultStoredDocument>? batch, out CacheBackingStore? home)
{
home = Home(stored.Descriptor.DocumentType);
if (home == null)
{
if (_stores.Count > 0)
return new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}.");
// No store, no file to judge, but the same rule: a record is replaced only by its own type's write.
return upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor)
? CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor)
: null;
}
if (home.IsReadOnly)
return new InvalidOperationException(
$"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}.");
return home.Refusal(stored.Key.Value, upsert ? stored.Descriptor : null)
?? (upsert && stored.Flattens ? FlattenRefusal(stored, batch) : null);
}
// A flatten writes its base chain's decoded view into a plain record, so it is refused when a record on that chain is not owned
// by its home store (the slots or drift that record's decode dropped would be lost): that ancestor's own refusal. The chain starts
// at the staged delta's base when the variant is flattened as it lands, else at the held variant's; a key the batch admits is
// judged by its own claim, and the walk follows its staged form.
private CultSchemaConflictException? FlattenRefusal(CultStoredDocument flatten, IReadOnlyDictionary<string, CultStoredDocument>? batch)
{
var delta = flatten.Variant ?? (_entries.TryGetValue(flatten.Key.Value, out var variant) ? variant.Variant : null);
var walked = new HashSet<string>(StringComparer.Ordinal) { flatten.Key.Value };
for (var key = delta?.BaseKey; key != null && walked.Add(key);)
{
CultStoredDocument? ancestor;
if (batch == null || !batch.TryGetValue(key, out ancestor))
{
if (_stores.Count > 0)
throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}.");
// No store, no file to judge, but the same rule: a record is replaced only by its own type's write.
if (upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor))
throw CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor);
return;
if (!_entries.TryGetValue(key, out ancestor))
return null;
if (Home(ancestor.Descriptor.DocumentType)?.Refusal(key, null) is { } refusal)
return refusal;
}
if (home.IsReadOnly)
throw new InvalidOperationException(
$"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}.");
home.RefuseUndeclared(stored.Key.Value, upsert ? stored.Descriptor : null);
homes.Add(home);
key = ancestor.Variant?.BaseKey;
}
return null;
}
private sealed class VariantPlan
@ -3937,11 +3998,21 @@ namespace GameCult.Caching
string version,
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
Func<Type, byte[], object> deserializePayload,
out CultSchemaMigrationReport report) =>
ToStoredDocument(record, version, catalog, (type, payload) => new CultDecoded(deserializePayload(type, payload), null), out report);
// The same, for a decode that notes the extra slots it read past (DeserializeMarked): the mark is turned into
// CultStoredDocument.UnreadSlots here, the one place that knows both the payload's marks and the catalog comparison.
internal CultStoredDocument ToStoredDocument(
CultPersistedRecord record,
string version,
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
Func<Type, byte[], CultDecoded> deserializePayload,
out CultSchemaMigrationReport report)
{
var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog);
report = resolution.Report;
var readWhole = report.IgnoredExtraSlots.Length == 0;
CultUnreadSlots? unread = null;
if (record.Variant != null)
{
var descriptor = resolution.Descriptor;
@ -3955,13 +4026,22 @@ namespace GameCult.Caching
.ToArray();
if (shed.Length > 0)
{
// A slot the schema diff already reported is not reported twice.
// A slot the schema diff already reported is not reported twice. One it did not report is a mark nothing explains:
// the variant is read-only and is never rewritten.
var unreported = shed.Except(report.IgnoredExtraSlots).ToArray();
if (unreported.Length > 0)
unread = new CultUnreadSlots(
descriptor.DocumentType.Name,
unreported.Max() + 1,
descriptor.RichMembers.Select(member => member.Slot + 1).DefaultIfEmpty(0).Max(),
1,
allAtDepthZero: true,
explained: false);
report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(unreported).OrderBy(slot => slot).ToArray();
report.Warnings = report.Warnings.Concat(unreported.Select(slot => new CultSchemaMigrationWarning
{
Code = "ignored_extra_slot",
Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush."
Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the variant is read-only."
})).ToArray();
}
@ -3977,16 +4057,17 @@ namespace GameCult.Caching
null)
{
StoredSchemaId = record.SchemaId,
StoredSchemaReadWhole = readWhole,
StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0,
UnreadSlots = unread,
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
StoredVersion = version
};
}
object document;
CultDecoded decoded;
try
{
document = deserializePayload(resolution.Descriptor.DocumentType, record.Payload);
decoded = deserializePayload(resolution.Descriptor.DocumentType, record.Payload);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
@ -3995,14 +4076,20 @@ namespace GameCult.Caching
for (var cause = exception; cause != null; cause = cause.InnerException) causes.Add(cause.Message);
throw new InvalidDataException($"Record '{record.Key}' (schema '{record.SchemaId}') cannot be decoded: {string.Join(" <- ", causes.Distinct())}", exception);
}
// Explained only when every mark is at depth 0 and the schema comparison under this id reported every slot the mark carries
// past the reader: consent reaches no slot above the highest reported (none when nothing was reported).
unread = decoded.UnreadSlots is { } marked
? marked.Explaining(marked.AllAtDepthZero && marked.SlotCount <= report.IgnoredExtraSlots.DefaultIfEmpty(-1).Max() + 1)
: null;
return new CultStoredDocument(
new CultRecordKey(record.Key),
record.StoredAt,
resolution.Descriptor,
document)
decoded.Document)
{
StoredSchemaId = record.SchemaId,
StoredSchemaReadWhole = readWhole,
StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0 && unread == null,
UnreadSlots = unread,
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
StoredVersion = version
};
@ -4041,22 +4128,37 @@ namespace GameCult.Caching
// catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file.
internal void RefuseUndeclared(string key, CultDocumentDescriptor? written)
{
var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
if (foreign != null)
throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) });
if (Entries.TryGetValue(key, out var held) && !HeldOwned(held, written))
throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) });
if (Refusal(key, written) is { } refusal) throw refusal;
}
// The refusal of a write (of the written type) or a removal (written null) of the key, or null when this store's view lets it
// through. The one answer behind RefuseUndeclared and CultCache.IsWritable, so the query and the write cannot disagree.
internal CultSchemaConflictException? Refusal(string key, CultDocumentDescriptor? written)
{
var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
if (foreign != null)
return Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) });
if (!Entries.TryGetValue(key, out var held) || HeldOwned(held, written))
return null;
return held.UnreadSlots is { Explained: false } unread
? UnreadSlotsRefusal(held, key, unread)
: Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) });
}
// The one ownership predicate of a held record. A mark nothing explains (anything but a depth-0 slot the catalog comparison
// reported ignored) keeps the record unowned whatever is declared: declaring the reader's own id consents to nothing.
private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) =>
written == null
? Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole
? !Unexplained(held) && (Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole)
: UpsertOwned(held, written);
private static bool Unexplained(CultStoredDocument held) => held.UnreadSlots is { Explained: false };
// Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held
// record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed
// cache's staging and a store-less cache's.
internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) =>
!Unexplained(held) &&
CultDocumentRegistry.WrittenOwns(
written.ToCatalogEntry(),
held.StoredSchemaId,
@ -4358,6 +4460,16 @@ namespace GameCult.Caching
refused[0].Key,
inner);
// The refusal of a write or removal of a record held read-only for the slots its payload carries past this build's. It names the
// type and the counts, never a slot value.
private CultSchemaConflictException UnreadSlotsRefusal(CultStoredDocument held, string key, CultUnreadSlots unread) => new(
$"{held.Descriptor.SchemaName} record {key} is read-only: {unread.TypeName} was read with {unread.SlotCount} slots where this build declares {unread.DeclaredCount} " +
$"({unread.Occurrences} object(s) in the record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.",
held.StoredSchemaId,
new[] { held.Descriptor.SchemaName },
key,
unread);
// A flush or commit succeeded: nothing staged is left, writes or checks.
protected void MarkFlushSucceeded()
{
@ -4461,6 +4573,9 @@ namespace GameCult.Caching
protected abstract byte[] SerializePayload(object document, Type documentType);
protected abstract object DeserializePayload(Type documentType, byte[] payload);
// A decode that also notes the slots it read past. A store kind that does not override it never marks.
internal virtual CultDecoded DeserializeMarked(Type documentType, byte[] payload) => new(DeserializePayload(documentType, payload), null);
public override string ToString() => FileInfo.FullName;
public override void PullAll() => Held(PullAllCore);
@ -4495,7 +4610,7 @@ namespace GameCult.Caching
continue;
}
var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report);
var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializeMarked, out var report);
reports.Add(report);
persisted[stored.Key.Value] = stored;
}
@ -4963,7 +5078,7 @@ namespace GameCult.Caching
var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal));
if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null)
return null;
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _);
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializeMarked, out _);
}
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:

View file

@ -5,7 +5,8 @@ namespace GameCult.Caching
/// <summary>
/// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the
/// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a
/// <see cref="CultForeignRecord"/>. Nothing is written, so the store stays as it was. It names the id, the schema names involved
/// <see cref="CultForeignRecord"/>, or a write or removal of a record held read-only because its payload carries slots past the
/// reader's declared slots (<see cref="UnreadSlots"/> is set). Nothing is written, so the store stays as it was. It names the id, the schema names involved
/// and a record key.
/// </summary>
public sealed class CultSchemaConflictException : InvalidOperationException
@ -27,8 +28,18 @@ namespace GameCult.Caching
RecordKey = recordKey;
}
/// <summary>The refusal of a record held read-only for the slots it carries past this build's: the same refusal, with the finding.</summary>
public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey, CultUnreadSlots unreadSlots)
: this(message, schemaId, schemaNames, recordKey)
{
UnreadSlots = unreadSlots;
}
public string SchemaId { get; }
public string[] SchemaNames { get; }
public string RecordKey { get; }
/// <summary>Set when the refusal is for unread slots; null for every other refusal.</summary>
public CultUnreadSlots? UnreadSlots { get; }
}
}

View file

@ -0,0 +1,54 @@
namespace GameCult.Caching
{
/// <summary>
/// What a store load found in a record's payload past the slots this build declares, at any depth: the first type read with extra
/// slots and how many slots it carried against how many it declares, and how many objects in the record did. It holds no slot
/// value, so it can be named in a refusal. A record carrying it is read-only (<see cref="CultCache.IsWritable"/>) unless every
/// mark is explained.
/// </summary>
public sealed class CultUnreadSlots
{
internal CultUnreadSlots(string typeName, int slotCount, int declaredCount, int occurrences, bool allAtDepthZero, bool explained)
{
TypeName = typeName;
SlotCount = slotCount;
DeclaredCount = declaredCount;
Occurrences = occurrences;
AllAtDepthZero = allAtDepthZero;
Explained = explained;
}
/// <summary>The first type read with extra slots.</summary>
public string TypeName { get; }
/// <summary>The slots that object carried.</summary>
public int SlotCount { get; }
/// <summary>The slots this build declares for that type: the array its own formatter writes for one instance.</summary>
public int DeclaredCount { get; }
/// <summary>The objects in the record that carried extra slots.</summary>
public int Occurrences { get; }
/// <summary>
/// True only when every mark is at depth 0 and the schema comparison under the id the record was stored under reported every
/// slot the mark carries past the reader (<see cref="SlotCount"/> is at most the highest reported slot plus one): the loss a
/// compatible-id declaration consents to (cultlib-gaps:ruling:slot-mark-consent). Any other mark leaves the record read-only
/// whatever is declared.
/// </summary>
public bool Explained { get; }
// Every mark the decode noted sat in the record's own object, the half of "explained" the payload decides.
internal bool AllAtDepthZero { get; }
internal CultUnreadSlots Explaining(bool explained) =>
new(TypeName, SlotCount, DeclaredCount, Occurrences, AllAtDepthZero, explained);
}
// A payload decoded by a store kind, with the extra slots the decode noted. A store kind that does not note them reports none.
internal readonly struct CultDecoded(object document, CultUnreadSlots? unreadSlots)
{
public object Document { get; } = document;
public CultUnreadSlots? UnreadSlots { get; } = unreadSlots;
}
}

View file

@ -4,6 +4,9 @@
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<!-- GameCult.Caching.MessagePack lets this check see CultUnionGuard.Inner (InternalsVisibleTo), which also shows it that
assembly's internal GeneratedMessagePackResolver; this project's own generated one is the type in use. -->
<NoWarn>$(NoWarn);CS0436</NoWarn>
</PropertyGroup>
<PropertyGroup Condition="'$(Aot)' == 'true'">

View file

@ -20,7 +20,8 @@ var documents = new (Type Type, object Value)[]
foreach (var (type, _) in documents)
{
var formatter = options.Resolver.GetFormatterDynamic(type)!.GetType();
// The guard wraps a measured type's formatter; the generated formatter it delegates to is the one that runs.
var formatter = CultUnionGuard.Inner(options.Resolver.GetFormatterDynamic(type)!).GetType();
if (formatter.Assembly.IsDynamic || !(formatter.FullName ?? "").Contains("GeneratedMessagePackResolver"))
return Fail($"{type.Name} is formatted by {formatter.FullName}, not a generated formatter");
}

View file

@ -10,6 +10,8 @@ using MessagePack.Formatters;
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))]
[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.ResolvedBagResolver))]
[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.RelayResolver))]
namespace GameCult.Caching.Tests
{

View file

@ -0,0 +1,639 @@
#nullable enable
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using GameCult.Caching.MessagePack;
using MessagePack;
using MessagePack.Formatters;
using NUnit.Framework;
using Surface = GameCult.Caching.Tests.UnionParityVectorTests.UnionSurface;
namespace GameCult.Caching.Tests
{
// cultlib-gaps:ruling:extra-trailing-slots and :slot-mark-consent. A record whose payload carries a slot past its reader's declared
// slots, at any depth, is held read-only: every staged write or removal of it is refused before anything is staged. A compatible-id
// declaration consents only to depth-0 slots the catalog comparison reports ignored under a listed compatible id.
public class SlotReadOnlyTests
{
private const string Extra = "extra-slot";
// Slot 1 and 2 are retired gaps.
[CultDocument("tests.slot_gap", "tests.slot_gap.v1")]
[MessagePackObject]
public sealed class GapDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(3)] public string Tail = string.Empty;
}
private static readonly CultDocumentRegistry SurfaceRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(Surface) });
private static readonly CultDocumentRegistry GapRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(GapDoc) });
private static readonly CultRecordKey Marked = new("marked");
private static readonly CultRecordKey Clean = new("clean");
private static readonly CultRecordKey Unheld = new("unheld");
private string _directory = string.Empty;
[SetUp]
public void CreateDirectory()
{
_directory = Path.Combine(Path.GetTempPath(), $"cultlib-slotro-{Guid.NewGuid():N}");
Directory.CreateDirectory(_directory);
}
[TearDown]
public void DeleteDirectory()
{
if (Directory.Exists(_directory)) Directory.Delete(_directory, recursive: true);
}
private static CultCache Open(string path, CultDocumentRegistry registry) =>
CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
private static byte[] Emit(WriterAction action)
{
var buffer = new ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
action(ref writer);
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
private delegate void WriterAction(ref MessagePackWriter writer);
// A store of two records of the given type, the marked one carrying the payload and the clean one an ordinary value.
private string Seed(CultDocumentRegistry registry, Type type, Func<object> clean, byte[] markedPayload, string name = "slots.cc")
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, registry))
{
cache.UpsertAsync(type, clean(), Marked).GetAwaiter().GetResult();
cache.UpsertAsync(type, clean(), Clean).GetAwaiter().GetResult();
cache.FlushAsync().GetAwaiter().GetResult();
}
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
snapshot.Records.Single(record => record.Key == Marked.Value).Payload = markedPayload;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
return path;
}
private string SeedSurface(string vector) =>
Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, VectorFiles.Committed("union-parity", vector));
private static CultStoredDocument Stored(CultCache cache, CultRecordKey key) => cache.GetStored(key)!;
// ---- a mark at every depth, with the type and both counts ----
[TestCase("extra-slot-record.msgpack", "UnionSurface", 8, 7, 1)]
[TestCase("extra-slot-nested-record.msgpack", nameof(UnionParityVectorTests.Composition), 3, 2, 1)]
[TestCase("extra-slot-arm.msgpack", nameof(UnionParityVectorTests.Box), 3, 2, 1)]
[TestCase("extra-slot-list-element.msgpack", nameof(UnionParityVectorTests.Text), 3, 2, 1)]
[TestCase("depth-skipped-at-limit.msgpack", "UnionSurface", 8, 7, 1)]
public void AMarkAtAnyDepthHoldsTheRecordReadOnly(string vector, string typeName, int slots, int declared, int occurrences)
{
var path = SeedSurface(vector);
using var cache = Open(path, SurfaceRegistry);
var unread = Stored(cache, Marked).UnreadSlots;
Assert.That(unread, Is.Not.Null, vector);
Assert.That(unread!.TypeName, Is.EqualTo(typeName), vector);
Assert.That(unread.SlotCount, Is.EqualTo(slots), vector);
Assert.That(unread.DeclaredCount, Is.EqualTo(declared), vector);
Assert.That(unread.Occurrences, Is.EqualTo(occurrences), vector);
Assert.That(unread.Explained, Is.False, "the reader's own id is not a compatible-id declaration: nothing explains the mark");
Assert.That(cache.IsWritable(Marked), Is.False, vector);
Assert.That(cache.Get<Surface>(Marked), Is.Not.Null, "a read-only record is still served");
Assert.That(Stored(cache, Clean).UnreadSlots, Is.Null);
Assert.That(cache.IsWritable(Clean), Is.True);
Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable");
}
// ---- every staged write and removal of a marked key is refused, nothing else is ----
private static void AssertSlotRefusal(TestDelegate act, string what, string type, int slots, int declared)
{
var refusal = Assert.Throws<CultSchemaConflictException>(act, what)!;
Assert.That(refusal.UnreadSlots, Is.Not.Null, what);
Assert.That(refusal.RecordKey, Is.EqualTo("marked"), what);
Assert.That(refusal.Message, Does.Contain("is read-only: " + type + " was read with " + slots + " slots where this build declares " + declared)
.And.Contain("record marked"), what);
Assert.That(refusal.Message, Does.Not.Contain(Extra), $"{what}: the refusal never echoes a slot value");
}
[Test]
public void EveryWriteAndRemovalOfAMarkedRecordIsRefusedAndAnotherKeyStillWrites()
{
var path = SeedSurface("extra-slot-nested-record.msgpack");
var before = File.ReadAllBytes(path);
using (var cache = Open(path, SurfaceRegistry))
{
var composition = nameof(UnionParityVectorTests.Composition);
AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Marked))), "upsert", composition, 3, 2);
AssertSlotRefusal(() => cache.Commit(batch => batch.UpsertVariant(Marked, Clean)), "variant", composition, 3, 2);
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "remove", composition, 3, 2);
AssertSlotRefusal(() => cache.Commit(batch => { batch.Upsert(new Surface { Id = "other" }, new CultRecordHandle<Surface>(Clean)); batch.Remove(Marked); }), "mixed batch", composition, 3, 2);
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused writes leave the store as it was");
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle<Surface>(Clean))), Is.True);
}
Assert.That(CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Count, Is.EqualTo(2));
}
// ---- consent ----
[Test]
public void ADeclaredIdDoesNotConsentToAnUnexplainedMark()
{
// The reader's own id is declared; the mark at depth 0 is in no compatible id's comparison: still read-only.
var path = SeedSurface("extra-slot-record.msgpack");
using var cache = Open(path, SurfaceRegistry);
AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Marked))), "depth 0 under the reader's own id", "UnionSurface", 8, 7);
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "removal", "UnionSurface", 8, 7);
}
// Every row of slot-outcomes.msgpack, from the committed files alone: the mark, Explained and IsWritable of each record of
// extra-slot-store.cc and extra-slot-consent.cc, and a removal of the key refused exactly where IsWritable is false.
[Test]
public void EverySlotOutcomeRowHoldsAndARemovalIsRefusedExactlyWhereTheRecordIsNotWritable()
{
var reader = new MessagePackReader(VectorFiles.Committed("union-parity", "slot-outcomes.msgpack"));
var rows = reader.ReadArrayHeader();
Assert.That(rows, Is.EqualTo(8));
for (var i = 0; i < rows; i++)
{
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(9));
var store = reader.ReadString()!;
var key = new CultRecordKey(reader.ReadString()!);
var writable = reader.ReadBoolean();
bool? explained = reader.TryReadNil() ? null : reader.ReadBoolean();
var type = reader.TryReadNil() ? null : reader.ReadString();
var counts = new int?[3];
for (var c = 0; c < 3; c++) counts[c] = reader.TryReadNil() ? null : reader.ReadInt32();
var (slots, declared, occurrences) = (counts[0], counts[1], counts[2]);
var flattenRefusedBy = reader.TryReadNil() ? null : reader.ReadString();
var row = $"{store} {key.Value}";
var path = Path.Combine(_directory, $"{i}-{store}");
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", store));
var before = File.ReadAllBytes(path);
using var cache = Open(path, UnionParityVectorTests.SlotReader(store));
var unread = Stored(cache, key).UnreadSlots;
if (explained == null)
{
Assert.That(unread, Is.Null, row);
}
else
{
Assert.That(unread, Is.Not.Null, row);
Assert.That((unread!.Explained, unread.TypeName, unread.SlotCount, unread.DeclaredCount, unread.Occurrences),
Is.EqualTo((explained.Value, type, slots, declared, occurrences)), row);
}
Assert.That(cache.IsWritable(key), Is.EqualTo(writable), row);
Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable");
if (flattenRefusedBy != null)
{
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch => batch.Flatten(key)), $"{row}: flatten")!;
Assert.That(refusal.RecordKey, Is.EqualTo(flattenRefusedBy), $"{row}: the refusal is the base's own");
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused flatten leaves the store as it was");
}
if (writable)
{
Assert.That(cache.Commit(batch => batch.Remove(key)), Is.True, row);
}
else
{
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch => batch.Remove(key)), row)!;
Assert.That(refusal.UnreadSlots, Is.Not.Null, row);
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused removal leaves the store as it was");
}
}
}
// Consent reaches only the slots the catalog comparison reports: the wide id's catalog reports slot 7, so a payload carrying
// slots 7 and 8 at depth 0 is not consented to, and a payload carrying slot 7 alone is.
[Test]
public void ConsentStopsAtTheSlotsTheCatalogReports()
{
var path = Path.Combine(_directory, "extra-slot-consent.cc");
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-consent.cc"));
var before = File.ReadAllBytes(path);
using var cache = Open(path, UnionParityVectorTests.SlotReader("extra-slot-consent.cc"));
var over = new CultRecordHandle<UnionParityVectorTests.SurfaceDeclaring>(new CultRecordKey("over-consent"));
AssertWriteAgrees(cache, over.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "over" }, over)));
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the refused write leaves the store as it was");
var consented = new CultRecordHandle<UnionParityVectorTests.SurfaceDeclaring>(new CultRecordKey("consented"));
AssertWriteAgrees(cache, consented.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "consented" }, consented)));
Assert.That(cache.IsWritable(consented.Key), Is.True);
}
// ---- carried verbatim ----
[Test]
public void AMarkedRecordIsCarriedByteForByteWhenAnotherRecordIsWritten()
{
var path = Path.Combine(_directory, "extra-slot-store.cc");
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-store.cc"));
var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var original = before.Records.Single(record => record.Key == Marked.Value);
using (var cache = Open(path, SurfaceRegistry))
{
Assert.That(cache.IsWritable(Marked), Is.False);
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle<Surface>(Clean))), Is.True);
}
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var carried = after.Records.Single(record => record.Key == Marked.Value);
Assert.That(carried.Payload, Is.EqualTo(original.Payload));
Assert.That(carried.SchemaId, Is.EqualTo(original.SchemaId));
Assert.That(carried.StoredAt, Is.EqualTo(original.StoredAt));
Assert.That(after.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString),
Is.EqualTo(before.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString)));
Assert.That(after.Records.Single(record => record.Key == Clean.Value).Payload, Is.Not.EqualTo(before.Records.Single(record => record.Key == Clean.Value).Payload));
}
// ---- slots short of the reader's, and gaps below the highest key, do not mark ----
[Test]
public void MissingTrailingSlotsDefaultAndAGapSlotDoesNotMark()
{
var short1 = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(1); w.Write("short"); });
var path = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, short1);
using (var cache = Open(path, GapRegistry))
{
Assert.That(cache.Get<GapDoc>(Marked)!.Tail, Is.EqualTo(string.Empty), "a missing trailing slot keeps its default");
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null);
Assert.That(cache.IsWritable(Marked), Is.True);
Assert.That(cache.Commit(batch => batch.Upsert(new GapDoc { Name = "short", Tail = "t" }, new CultRecordHandle<GapDoc>(Marked))), Is.True);
}
var gapped = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(4); w.Write("gap"); w.WriteNil(); w.Write("in a retired slot"); w.Write("tail"); });
var gapPath = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, gapped, "gap.cc");
using (var cache = Open(gapPath, GapRegistry))
{
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, "a non-nil value in a retired gap below the highest key does not mark");
Assert.That(cache.IsWritable(Marked), Is.True);
}
}
// ---- the catalog's consent still works, and is reported as explained ----
[CultDocument("tests.slot_wide", "tests.slot_wide.v1")]
[MessagePackObject]
public sealed class WideDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public string Colour = string.Empty;
[Key(2)] public string Extra = string.Empty;
}
[CultDocument("tests.slot_wide", "tests.slot_wide.v2")]
[MessagePackObject]
public sealed class NarrowDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public string Colour = string.Empty;
}
[Test]
public void ADepthZeroSlotTheCatalogReportsIsExplainedAndTheUndeclaredRecordStaysUnwritable([Values] bool directory)
{
var path = Path.Combine(_directory, directory ? "store" : "store.cc");
CacheBackingStore Store() => directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path);
using (var writer = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) })))
{
writer.AddBackingStore(Store());
Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "w", Extra = "kept?" }, new CultRecordHandle<WideDoc>(Marked))), Is.True);
}
using var reader = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) }));
reader.AddBackingStore(Store());
var unread = Stored(reader, Marked).UnreadSlots;
Assert.That(unread, Is.Not.Null, "the store kind decodes through the marking path");
Assert.That(unread!.Explained, Is.True, "depth 0 and reported by the catalog comparison");
Assert.That(unread.TypeName, Is.EqualTo(nameof(NarrowDoc)));
Assert.That(unread.SlotCount, Is.EqualTo(3));
Assert.That(unread.DeclaredCount, Is.EqualTo(2));
Assert.That(reader.IsWritable(Marked), Is.False, "the wide id is undeclared: unowned, though the mark is explained");
Assert.Throws<CultSchemaConflictException>(() => reader.Commit(batch => batch.Remove(Marked)));
}
// ---- declared slots are the slots the type's own formatter writes ----
// MessagePack writes a non-public [Key] member of an AllowPrivate type: 2 slots, where a public-member count says 1.
[MessagePackObject(AllowPrivate = true)]
public sealed class PrivInner
{
[Key(0)] public string A = string.Empty;
[Key(1)] internal string B = string.Empty;
}
// A [Key] getter that throws on an uninitialized instance; the parameterless constructor's instance is written instead.
[MessagePackObject]
public sealed class Counted
{
private List<string> _items = new();
[Key(0)] public int Count { get => _items.Count; set => _items = Enumerable.Repeat(string.Empty, value).ToList(); }
}
// The same getter and no parameterless constructor: no instance can be written, so the guard leaves the type alone.
[MessagePackObject]
public sealed class Uncounted
{
private readonly List<string> _items;
[SerializationConstructor] public Uncounted(int count) => _items = Enumerable.Repeat(string.Empty, count).ToList();
[Key(0)] public int Count => _items.Count;
}
// A type naming its own formatter owns its layout: one [Key] slot, but the formatter writes the item count and each item.
[MessagePackObject]
[MessagePackFormatter(typeof(BagFormatter))]
public sealed class Bag
{
[Key(0)] public List<string> Items = new();
}
public sealed class BagFormatter : IMessagePackFormatter<Bag?>
{
public void Serialize(ref MessagePackWriter writer, Bag? value, MessagePackSerializerOptions options)
{
if (value == null) { writer.WriteNil(); return; }
writer.WriteArrayHeader(1 + value.Items.Count);
writer.Write(value.Items.Count);
foreach (var item in value.Items) writer.Write(item);
}
public Bag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
if (reader.TryReadNil()) return null;
var count = reader.ReadArrayHeader();
reader.ReadInt32();
var bag = new Bag();
for (var i = 1; i < count; i++) bag.Items.Add(reader.ReadString()!);
return bag;
}
}
// A [MessagePackObject] type an assembly resolver formats as [count, items...], with no [MessagePackFormatter] attribute: the
// resolver owns its layout.
[MessagePackObject]
public sealed class ResolvedBag
{
[Key(0)] public List<string> Items = new();
}
public sealed class ResolvedBagResolver : IFormatterResolver
{
public static readonly IFormatterResolver Instance = new ResolvedBagResolver();
private static readonly IMessagePackFormatter<ResolvedBag?> Formatter = new ResolvedBagFormatter();
public IMessagePackFormatter<T>? GetFormatter<T>() => typeof(T) == typeof(ResolvedBag) ? (IMessagePackFormatter<T>)Formatter : null;
private sealed class ResolvedBagFormatter : IMessagePackFormatter<ResolvedBag?>
{
private static readonly BagFormatter Bag = new();
public void Serialize(ref MessagePackWriter writer, ResolvedBag? value, MessagePackSerializerOptions options) =>
Bag.Serialize(ref writer, value == null ? null : new Bag { Items = value.Items }, options);
public ResolvedBag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) =>
Bag.Deserialize(ref reader, options) is { } bag ? new ResolvedBag { Items = bag.Items } : null;
}
}
// A type an assembly resolver claims by handing out MessagePack's own [Key]-layout formatter: still measured.
[MessagePackObject]
public sealed class Relayed
{
[Key(0)] public string A = string.Empty;
}
public sealed class RelayResolver : IFormatterResolver
{
public static readonly IFormatterResolver Instance = new RelayResolver();
public IMessagePackFormatter<T>? GetFormatter<T>() =>
typeof(T) == typeof(Relayed) ? global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter<T>() : null;
}
// A [MessagePackObject] collection: MessagePack writes it as its items, not by [Key].
[MessagePackObject]
public class TagList : List<string> { }
[CultDocument("tests.slot_measured", "tests.slot_measured.v1")]
[MessagePackObject]
public sealed class MeasuredDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public PrivInner? Private;
[Key(2)] public Counted? Counted;
[Key(3)] public Uncounted? Uncounted;
[Key(4)] public Bag? Bag;
[Key(5)] public ResolvedBag? Resolved;
[Key(6)] public TagList? Tags;
[Key(7)] public Relayed? Relayed;
}
private static readonly CultDocumentRegistry MeasuredRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(MeasuredDoc) });
private static MeasuredDoc Measured(string member) => member switch
{
"private" => new MeasuredDoc { Name = member, Private = new PrivInner { A = "a", B = "bee" } },
"constructed" => new MeasuredDoc { Name = member, Counted = new Counted { Count = 2 } },
"unmeasurable" => new MeasuredDoc { Name = member, Uncounted = new Uncounted(2) },
"own-formatter" => new MeasuredDoc { Name = member, Bag = new Bag { Items = { "x", "y", "z" } } },
"resolver-formatted" => new MeasuredDoc { Name = member, Resolved = new ResolvedBag { Items = { "x", "y", "z" } } },
"collection" => new MeasuredDoc { Name = member, Tags = new TagList { "a", "b", "c" } },
_ => throw new ArgumentOutOfRangeException(nameof(member), member, null)
};
[TestCase("private", TestName = "APrivateKeyedMemberDoesNotMark")]
[TestCase("constructed", TestName = "ATypeMeasuredThroughItsConstructorDoesNotMark")]
[TestCase("unmeasurable", TestName = "AnUnmeasurableTypeIsNotGuarded")]
[TestCase("own-formatter", TestName = "ATypeWithItsOwnFormatterIsNotMeasuredByKeys")]
[TestCase("resolver-formatted", TestName = "ATypeAResolverFormatsIsNotMeasuredByKeys")]
[TestCase("collection", TestName = "AMessagePackObjectCollectionIsNotMeasuredByKeys")]
public void ARecordWrittenByItsOwnTypeDoesNotMark(string member)
{
var path = Path.Combine(_directory, "measured.cc");
using (var cache = Open(path, MeasuredRegistry))
{
Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle<MeasuredDoc>(Marked))), Is.True);
}
using (var cache = Open(path, MeasuredRegistry))
{
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, member);
Assert.That(cache.IsWritable(Marked), Is.True, member);
Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle<MeasuredDoc>(Marked))), Is.True, member);
Assert.That(cache.Commit(batch => batch.Remove(Marked)), Is.True, member);
}
}
// One slot past what the formatter writes still marks, with the written count as declared.
[Test]
public void AnExtraSlotPastTheWrittenCountStillMarks()
{
var privatePayload = Emit((ref MessagePackWriter w) =>
{
w.WriteArrayHeader(5); w.Write("p");
w.WriteArrayHeader(3); w.Write("a"); w.Write("bee"); w.Write(Extra);
w.WriteNil(); w.WriteNil(); w.WriteNil();
});
var path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, privatePayload, "private.cc");
using (var cache = Open(path, MeasuredRegistry))
{
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "AllowPrivate type", nameof(PrivInner), 3, 2);
Assert.That(cache.IsWritable(Clean), Is.True);
}
var countedPayload = Emit((ref MessagePackWriter w) =>
{
w.WriteArrayHeader(5); w.Write("c"); w.WriteNil();
w.WriteArrayHeader(2); w.Write(0); w.Write(Extra);
w.WriteNil(); w.WriteNil();
});
path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, countedPayload, "counted.cc");
using (var cache = Open(path, MeasuredRegistry))
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "measured through its constructor", nameof(Counted), 2, 1);
// A resolver that hands out MessagePack's own [Key] formatter exempts nothing.
var relayedPayload = Emit((ref MessagePackWriter w) =>
{
w.WriteArrayHeader(8); w.Write("r");
for (var i = 0; i < 6; i++) w.WriteNil();
w.WriteArrayHeader(2); w.Write("a"); w.Write(Extra);
});
path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, relayedPayload, "relayed.cc");
using (var cache = Open(path, MeasuredRegistry))
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "relayed by a consumer resolver", nameof(Relayed), 2, 1);
}
// ---- a flatten writes its base chain's decoded view: refused when a record on the chain is not owned ----
private static readonly CultRecordKey Base = new("base");
private static readonly CultRecordKey Var = new("var");
private static readonly CultRecordKey Fresh = new("fresh");
// A store holding plain record base and its variant var, base then given the payload.
private string SeedVariant(byte[]? basePayload, string name)
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, SurfaceRegistry))
{
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "base" }, new CultRecordHandle<Surface>(Base))), Is.True);
Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True);
}
if (basePayload == null) return path;
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
snapshot.Records.Single(record => record.Key == Base.Value).Payload = basePayload;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
return path;
}
// A variant overrides its name.
private static CultVariantOverride[] Named(CultCache cache, CultRecordKey key) => new[] { cache.Override<Surface>(nameof(Surface.Id), key.Value) };
private static void AssertBaseRefusal(TestDelegate act, string what, bool unreadSlots)
{
var refusal = Assert.Throws<CultSchemaConflictException>(act, what)!;
Assert.That(refusal.RecordKey, Is.EqualTo(Base.Value), $"{what}: the refusal is the base's own");
Assert.That(refusal.UnreadSlots, unreadSlots ? Is.Not.Null : Is.Null, what);
}
[Test]
public void AFlattenThroughABaseHeldForUnreadSlotsIsRefusedAndTheVariantStillWritesItsDelta()
{
var nested = VectorFiles.Committed("union-parity", "extra-slot-nested-record.msgpack");
var path = SeedVariant(nested, "flatten.cc");
var before = File.ReadAllBytes(path);
using (var cache = Open(path, SurfaceRegistry))
{
Assert.That(cache.IsWritable(Base), Is.False);
Assert.That(cache.IsWritable(Var), Is.True, "the variant persists only its delta");
AssertBaseRefusal(() => cache.Commit(batch => batch.Flatten(Var)), "Commit(Flatten)", unreadSlots: true);
AssertBaseRefusal(() => cache.FlattenAsync(Var).GetAwaiter().GetResult(), "FlattenAsync", unreadSlots: true);
AssertBaseRefusal(() => cache.Commit(batch => { batch.UpsertVariant(Fresh, Base, Named(cache, Fresh)); batch.Flatten(Fresh); }), "a new variant flattened as it lands", unreadSlots: true);
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused flattens leave the store as it was");
Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True, "a variant write persists only its delta");
Assert.That(cache.Commit(batch => batch.Remove(Var)), Is.True);
}
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
Assert.That(after.Records.Single(record => record.Key == Base.Value).Payload, Is.EqualTo(nested), "the base's bytes stay");
}
[Test]
public void AFlattenThroughABaseUnderAnUndeclaredDroppingIdIsRefused()
{
var path = Path.Combine(_directory, "drift.cc");
using (var writer = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) })))
Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "base", Extra = "kept?" }, new CultRecordHandle<WideDoc>(Base))), Is.True);
using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) }));
Assert.That(reader.IsWritable(Base), Is.False);
Assert.That(reader.Commit(batch => batch.UpsertVariant(Var, Base, new[] { reader.Override<NarrowDoc>(nameof(NarrowDoc.Name), Var.Value) })), Is.True, "a variant over the unowned base persists only its delta");
var before = File.ReadAllBytes(path);
AssertBaseRefusal(() => reader.Commit(batch => batch.Flatten(Var)), "flatten over undeclared drift", unreadSlots: false);
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before));
Assert.That(reader.Commit(batch => batch.Remove(Var)), Is.True);
}
// ---- IsWritable is the write's own answer ----
// False exactly where the write throws.
private static void AssertWriteAgrees(CultCache cache, CultRecordKey key, Action write)
{
var writable = cache.IsWritable(key);
Exception? thrown = null;
try { write(); }
catch (InvalidOperationException exception) { thrown = exception; }
Assert.That(thrown == null, Is.EqualTo(writable), $"IsWritable({key.Value}) is {writable}; the write {(thrown == null ? "landed" : "threw: " + thrown.Message)}");
}
[Test]
public void IsWritableIsFalseInAReadOnlyStoreAndAFlattenThereIsNotRefusedForItsCleanBase()
{
var path = SeedVariant(null, "readonly.cc");
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = SurfaceRegistry, ReadOnly = true });
Assert.That(cache.IsWritable(Base), Is.False);
AssertWriteAgrees(cache, Base, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Base))));
AssertWriteAgrees(cache, Var, () => cache.Commit(batch => batch.Remove(Var)));
var refusal = Assert.Throws<InvalidOperationException>(() => cache.Commit(batch => batch.Flatten(Var)))!;
Assert.That(refusal, Is.Not.InstanceOf<CultSchemaConflictException>(), "nothing on the chain is lost: the refusal is the store's read-only one");
Assert.That(refusal.Message, Does.Contain("read-only backing store"));
}
// A key the cache does not hold, but a store carries as a foreign record: the store's removal check refuses it.
[Test]
public void IsWritableIsFalseForAnUnheldKeyAStoreListsForeign()
{
var path = Path.Combine(_directory, "foreign.cc");
using (var writer = Open(path, GapRegistry))
Assert.That(writer.Commit(batch => batch.Upsert(new GapDoc { Name = "g" }, new CultRecordHandle<GapDoc>(Marked))), Is.True);
using var cache = Open(path, SurfaceRegistry);
Assert.That(cache.GetStored(Marked), Is.Null, "the cache does not hold a record no registered type reads");
Assert.That(cache.IsWritable(Marked), Is.False);
AssertWriteAgrees(cache, Marked, () => cache.Commit(batch => batch.Remove(Marked)));
Assert.That(cache.IsWritable(Unheld), Is.True);
}
[Test]
public void IsWritableIsFalseForARecordNoAttachedStoreIsHomeTo()
{
using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Surface), typeof(GapDoc) }));
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "memory" }, new CultRecordHandle<Surface>(Clean))), Is.True);
Assert.That(cache.IsWritable(Clean), Is.True, "a store-less cache writes its own records");
cache.AddBackingStore(new SingleFileMessagePackBackingStore(Path.Combine(_directory, "gaps.cc")), typeof(GapDoc));
Assert.That(cache.IsWritable(Clean), Is.False);
AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "x" }, new CultRecordHandle<Surface>(Clean))));
AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Remove(Clean)));
Assert.That(cache.IsWritable(Unheld), Is.True, "an unheld key no store lists foreign is writable");
}
}
}

View file

@ -252,7 +252,7 @@ namespace GameCult.Caching.Tests
private static Emit ImageArm(string id, string uri, string alt) => ArmOf(5, Str(id), Str(uri), Nil, Str(alt));
// The constructed value's payload, with a part of the structure replaceable to build a refusal or an extra-slot case.
private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, Emit? trailing = null)
private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, params Emit[] trailing)
{
extra ??= new Extra();
root ??= BoxArm("0000000000c0", Slots(
@ -264,7 +264,7 @@ namespace GameCult.Caching.Tests
outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3")));
crafted ??= ArmOf(1, Str("gear"), Num(4));
var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil };
return Raw(Slots(trailing != null ? slots.Append(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots));
return Raw(Slots(trailing.Length > 0 ? slots.Concat(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots));
}
private static byte[] SerializeValue(UnionSurface value) => CultDocumentMessagePackSerialization.SerializeUntyped(value, typeof(UnionSurface), Registry);
@ -338,6 +338,73 @@ namespace GameCult.Caching.Tests
[MessagePackObject] public sealed class Floor2 : UnitOutcome { }
[MessagePackObject] public sealed class Ceiling2 : UnitOutcome { }
// The document a unit arm is placed in for a store: [Name, Outcome].
[CultDocument("tests.slot_unit", "tests.slot_unit.v1")]
[MessagePackObject]
public sealed class UnitDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public UnitOutcome? Outcome;
}
// ---- the slot stores: extra-slot-store.cc, extra-slot-consent.cc and their outcomes as data ----
// The id extra-slot-consent.cc's records are stored under: UnionSurface's catalog entry with one more member, slot 7.
internal const string WideId = "tests.union_surface.wide";
// UnionSurface's members under a new version, listing the wide id compatible: the consent vector's reader.
[CultDocument("tests.union_surface", "tests.union_surface.v2", CompatibleSchemaIds = new[] { WideId })]
[MessagePackObject]
public sealed class SurfaceDeclaring
{
[Key(0)] [CultName] public string Id = string.Empty;
[Key(1)] public int Version;
[Key(2)] public Node? Root;
[Key(3)] public List<Outcome> Outcomes = new();
[Key(4)] public Item? Part;
[Key(5)] public CraftedItem? Crafted;
[Key(6)] public Node? Missing;
}
// The registry each slot store is read with.
internal static CultDocumentRegistry SlotReader(string store) => store switch
{
"extra-slot-store.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(UnionSurface), typeof(UnitDoc) }),
"extra-slot-consent.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(SurfaceDeclaring) }),
_ => throw new ArgumentOutOfRangeException(nameof(store), store, null)
};
// One row of slot-outcomes.msgpack: what the store's reader holds at the key. The mark fields are null for an unmarked record;
// FlattenRefusedBy is the key whose refusal refuses a Flatten of the record, null when the record is no variant or flattens.
private sealed record SlotOutcome(string Store, string Key, bool Writable, bool? Explained, string? Type, int? Slots, int? Declared, int? Occurrences, string? FlattenRefusedBy = null);
private static readonly SlotOutcome[] SlotOutcomes =
{
new("extra-slot-store.cc", "marked", false, false, nameof(Composition), 3, 2, 1),
new("extra-slot-store.cc", "clean", true, null, null, null, null, null),
new("extra-slot-store.cc", "unit-arm", false, false, nameof(Denied2), 1, 0, 1),
new("extra-slot-store.cc", "marked-variant", true, null, null, null, null, null, "marked"),
new("extra-slot-consent.cc", "consented", true, true, nameof(SurfaceDeclaring), 8, 7, 1),
new("extra-slot-consent.cc", "over-consent", false, false, nameof(SurfaceDeclaring), 9, 7, 1),
new("extra-slot-consent.cc", "nested", false, false, nameof(SurfaceDeclaring), 8, 7, 2),
new("extra-slot-consent.cc", "clean", true, null, null, null, null, null),
};
private static byte[] SlotOutcomesFile() => Raw((ref MessagePackWriter w) =>
{
w.WriteArrayHeader(SlotOutcomes.Length);
foreach (var row in SlotOutcomes)
{
w.WriteArrayHeader(9);
w.Write(row.Store); w.Write(row.Key); w.Write(row.Writable);
if (row.Explained is { } explained) w.Write(explained); else w.WriteNil();
if (row.Type is { } type) w.Write(type); else w.WriteNil();
foreach (var count in new[] { row.Slots, row.Declared, row.Occurrences })
if (count is { } value) w.Write(value); else w.WriteNil();
if (row.FlattenRefusedBy is { } refusedBy) w.Write(refusedBy); else w.WriteNil();
}
});
private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly);
// The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes.
@ -413,6 +480,79 @@ namespace GameCult.Caching.Tests
}
}
// A single-file store of the records, each written as its type's value with the store's clock pinned as union-surface.cc's, then
// its payload replaced by bytes no current writer produces; each variant is written over its base (overriding the name to its
// key) before the payloads are replaced; edit changes the snapshot further before it is serialized.
private static byte[] SlotStore(CultDocumentRegistry registry, (string Key, object Value, byte[] Payload)[] records, Action<CultPersistedStoreSnapshot>? edit = null,
params (string Key, string Base)[] variants)
{
var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}");
Directory.CreateDirectory(work);
try
{
var scratch = Path.Combine(work, "slots.cc");
using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = registry, StoreFlushOnDispose = true }))
{
cache.Clock = () => StoredAt;
foreach (var (key, value, _) in records)
cache.UpsertAsync(value.GetType(), value, new CultRecordKey(key)).GetAwaiter().GetResult();
foreach (var (key, baseKey) in variants)
Assert.That(cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), new CultRecordKey(baseKey),
new[] { cache.Override(typeof(UnionSurface), nameof(UnionSurface.Id), key) })), Is.True);
cache.FlushAsync().GetAwaiter().GetResult();
}
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(scratch));
foreach (var (key, _, payload) in records)
snapshot.Records.Single(record => record.Key == key).Payload = payload;
edit?.Invoke(snapshot);
return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
}
finally
{
Directory.Delete(work, recursive: true);
}
}
// The store the slot-readonly tests open and the other runtimes read: "marked" holds the extra-slot-nested-record payload,
// "clean" the ordinary value, "unit-arm" a tests.slot_unit document ["unit-arm", [2, ["extra-slot"]]], and "marked-variant" a
// variant of "marked" overriding only its Id: writable, but a flatten of it is refused for its base.
[Test]
public void ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord()
{
Check("extra-slot-store.cc", SlotStore(SlotReader("extra-slot-store.cc"), new (string, object, byte[])[]
{
("marked", Constructed(), Committed("extra-slot-nested-record.msgpack")),
("clean", Constructed(), Surface()),
("unit-arm", new UnitDoc { Name = "unit-arm", Outcome = new Denied2() }, Raw(Slots(Str("unit-arm"), ArmOf(2, ExtraSlot)))),
}, null, ("marked-variant", "marked")));
}
// The consent store: every record under the wide id, whose catalog entry is UnionSurface's 7 members plus slot 7. Its reader
// lists the wide id compatible, so the comparison reports slot 7 ignored; only a depth-0 mark within that slot is consented to.
[Test]
public void ExtraSlotConsentVectorHoldsTheConsentCases()
{
Check("extra-slot-consent.cc", SlotStore(Registry, new (string, object, byte[])[]
{
("consented", Constructed(), Surface(trailing: ExtraSlot)),
("over-consent", Constructed(), Surface(trailing: new[] { ExtraSlot, ExtraSlot })),
("nested", Constructed(), Surface(extra: new Extra(Composition: true), trailing: ExtraSlot)),
("clean", Constructed(), Surface()),
}, snapshot =>
{
var entry = snapshot.SchemaCatalog.Single();
var last = entry.Members.OrderBy(member => member.Slot).Last();
entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 7, MemberName = "Wider", TypeName = last.TypeName }).ToArray();
entry.SchemaId = WideId;
foreach (var record in snapshot.Records) record.SchemaId = WideId;
}));
}
// The outcomes of both slot stores as data; SlotReadOnlyTests asserts every row from the committed files.
[Test]
public void SlotOutcomesVectorMatches() => Check("slot-outcomes.msgpack", SlotOutcomesFile());
// The value names every arm shape the vectors exist to hold other runtimes to.
[Test]
public void ConstructedValueExercisesEveryArmShape()
@ -475,7 +615,7 @@ namespace GameCult.Caching.Tests
else
{
Assert.That(outcome, Is.EqualTo("read-only"), file);
// The record is read whole; what makes it read-only is the slot rule of the slot-tolerance cut, not asserted here.
// The extra slot is read past; that the record then loads read-only is asserted in SlotReadOnlyTests, store by store.
Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past");
}
}
@ -781,7 +921,8 @@ namespace GameCult.Caching.Tests
Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x91 }), $"switch {off ?? "unset"} rewrites nothing");
File.Delete(path);
foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles })
foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles,
ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord, ExtraSlotConsentVectorHoldsTheConsentCases, SlotOutcomesVectorMatches })
body();
}

View file

@ -513,7 +513,7 @@ namespace GameCult.Caching.Tests
}
[Test]
public void AnOverrideOfASlotTheTypeNeverHadIsIgnoredOnLoadAsAPlainRecordIgnoresAnExtraSlot()
public void AnOverrideOfASlotTheTypeNeverHadIsReadPastOnLoadAsAPlainRecordReadsPastAnExtraSlot()
{
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
CultVariantOverride.Set(99, new byte[] { 0x01 }),
@ -1039,7 +1039,7 @@ namespace GameCult.Caching.Tests
}
[Test]
public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndStaysStoredUntilTheVariantIsWritten()
public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndHoldsTheVariantReadOnly()
{
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
CultVariantOverride.Set(99, new byte[] { 0x01 }),
@ -1059,14 +1059,22 @@ namespace GameCult.Caching.Tests
{
Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 }));
Assert.That(reopened.Get<VariantGear>(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant"));
reopened.Commit(batch => batch.UpsertVariant(new CultRecordKey("hand-variant"), new CultRecordKey("hand-base"), new[]
var variant = new CultRecordKey("hand-variant");
Assert.That(reopened.IsWritable(variant), Is.False, "an override of a slot the type lacks, which the catalog did not report, is a mark nothing explains");
var overrides = new[]
{
reopened.Override<VariantGear>(nameof(VariantGear.Name), "hand variant"),
reopened.Override<VariantGear>(nameof(VariantGear.Code), "hand-code")
}));
};
var rewrite = Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.UpsertVariant(variant, new CultRecordKey("hand-base"), overrides)))!;
Assert.That(rewrite.UnreadSlots, Is.Not.Null);
Assert.That(rewrite.UnreadSlots!.TypeName, Is.EqualTo(nameof(VariantGear)));
Assert.That(rewrite.UnreadSlots.SlotCount, Is.EqualTo(100));
Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.Flatten(variant)));
Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.Remove(variant)));
}
Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "a write of the variant stores only what it was given");
Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2, 99 }), "the refused writes left the variant as stored");
}
private static int[] OverriddenSlotsOn(string path, string key) =>

View file

@ -48,19 +48,93 @@ with one part replaced.
## Extra trailing slot (outcome `read-only`)
Each payload is `union-surface.msgpack` with one slot appended past the reader's last declared slot of one
record (ruling `cultlib-gaps:ruling:extra-trailing-slots`: the record is read, read-only at any depth, every write
of it refused). A reader decodes the value of `union-surface.msgpack` from each; `detail` says where the slot is.
Rulings `cultlib-gaps:ruling:extra-trailing-slots` and `cultlib-gaps:ruling:slot-mark-consent`: a payload carrying a
slot past its reader's declared slots, in any object at any depth, is read, held read-only, and every write or
removal of it is refused before anything is staged. The first four payloads are `union-surface.msgpack` with the
string `"extra-slot"` appended as one more slot of one object; a reader decodes the value of `union-surface.msgpack`
from each. A unit arm with an extra slot is pinned only as a store record, `unit-arm` in `extra-slot-store.cc`
(below): `outcomes.msgpack` keeps the nine rows the leaf walkers count.
| File | detail | Record with the extra slot |
|---|---|---|
| `extra-slot-record.msgpack` | record | the document (slot 7) |
| `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` |
| `extra-slot-arm.msgpack` | arm | the Root Box arm |
| `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots |
An object's declared slots are the slots its type's own formatter writes for one instance (for these fixtures,
the highest key plus one); a reader derives them from the layout it writes, never from a list of public members.
Only a type the runtime writes by its own positional `[Key]` layout is checked. Exempt, as in the C# reference
(`cultlib-gaps:ruling:leaf-acceptance-follows-reference`): a type the runtime formats some other way (a
user codec or formatter, a hand-written serializer, a resolver ahead of the runtime's own that supplies a layout
of its own, a collection type written as its items; a resolver that hands out the runtime's own positional
layout exempts nothing), and a type no instance of which the runtime can write for measuring
(`cultlib-gaps:ruling:slot-unmeasurable-type`). Neither is ever marked. No vector carries an exempt type: each
runtime pins its own exemption with its own fixture.
Only an array longer than that marks: a short array
(missing trailing slots) keeps the defaults and is writable, and a non-nil value in a retired slot below the highest
key (Image's slot 2) does not mark. A union adds no depth: an arm's array is counted against the arm's own slots.
The C# reference reads these today and its slot-tolerance cut is what makes them read-only; this cut asserts only
that the value is read.
| File | detail | Object with the extra slot | Depth | Mark: type, slots carried, slots declared |
|---|---|---|---|---|
| `extra-slot-record.msgpack` | record | the document (slot 7) | 0 | UnionSurface, 8, 7 |
| `extra-slot-nested-record.msgpack` | nested-record | Composition `comp` | nested | Composition, 3, 2 |
| `extra-slot-arm.msgpack` | arm | the Root Box arm | nested | Box, 3, 2 |
| `extra-slot-list-element.msgpack` | list-element | Text `s1`, an element of Use.Slots | nested | Text, 3, 2 |
| record `unit-arm` of `extra-slot-store.cc` | unit-arm | Denied2 in `[2, ["extra-slot"]]` (union `UnitOutcome`, key 2), slot 1 of the document | 1 | Denied2, 1, 0 |
| `depth-skipped-at-limit.msgpack` (bounds file) | depth | the document (slot 7, a value of nested arrays) | 0 | UnionSurface, 8, 7 |
Each payload marks exactly one object. Four rows of `outcomes.msgpack` are `read-only` (record, nested-record, arm,
list-element); the fifth `read-only` payload is in `bounds-outcomes.msgpack`, and the unit arm's outcome is its row
in `slot-outcomes.msgpack`. The payload nesting check
runs before the mark: `depth-skipped-past-limit.msgpack` is refused, never marked.
### What a runtime asserts for a `read-only` payload
Load it as a store record under the reader's own schema id (the C# reference puts each UnionSurface payload in a
`tests.union_surface` store; the unit arm is already a store record, `unit-arm` of `extra-slot-store.cc`). Then:
1. the value is read past the extra slot;
2. the record reports the mark as in the table: the first type read with extra slots, the slots it carried and the
slots the reader declares, and the count of objects that carried extra slots (1 here);
3. the record is not writable (C#: `CultCache.IsWritable(key)` false), and a key no store holds is writable;
4. a replacing write, a variant write and a removal of the key, and a flatten of a variant whose base chain holds
the marked record, are refused before anything is staged,
with the runtime's schema conflict refusal carrying the mark; its message names the type and both counts and
never contains `extra-slot`, the slot's value;
5. a write to another key of the same store succeeds.
Consent: a mark is explained only when it is at depth 0 and the schema comparison of the record's stored id against
the reader reported ignored extra slots covering it: the record is stored under an id the reader lists compatible,
that id's catalog entry has more members, and the mark's slot count is at most the highest reported slot plus one.
Declaring that id consents to an explained mark only. Every mark in the table above is unexplained (stored under
the reader's own id), so each record stays read-only whatever is declared.
## Slot stores (`slot-outcomes.msgpack`)
Two single-file stores, each record written as a value and its payload then replaced, the store's clock pinned to
2026-01-01T00:00:00Z as `union-surface.cc`'s:
- `extra-slot-store.cc`, read with `tests.union_surface` (schema `tests.union_surface.v1`) and `tests.slot_unit`
(schema `tests.slot_unit.v1`, a document `[Name, Outcome: UnitOutcome]`). `marked` is
`extra-slot-nested-record.msgpack`; `clean` is `union-surface.msgpack`; `unit-arm` is the `tests.slot_unit`
document `["unit-arm", [2, ["extra-slot"]]]`, whose Denied2 arm sits at depth 1 (the document is depth 0; a union
adds none); `marked-variant` is a variant of `marked` overriding only `Id` (slot 0) to `"marked-variant"`.
- `extra-slot-consent.cc`: every record is stored under `tests.union_surface.wide`, whose catalog entry is
`tests.union_surface`'s seven members plus slot 7 (`Wider`). Read it with a `tests.union_surface` type of schema
`tests.union_surface.v2` (the reference names it `SurfaceDeclaring`) that lists `tests.union_surface.wide`
compatible, so the comparison reports slot 7 ignored. `consented` is `union-surface.msgpack` plus one root slot
(slot 7); `over-consent` plus two (slots 7 and 8, past the highest reported); `nested` is
`extra-slot-nested-record.msgpack` plus one root slot; `clean` is `union-surface.msgpack`.
`slot-outcomes.msgpack` is a second outcomes file, as `bounds-outcomes.msgpack` is, so readers of `outcomes.msgpack`
stay green. It is an array of rows `[store, key, writable, explained, type, slots, declared, occurrences, flattenRefusedBy]`,
one per record of both stores: `writable` is what `IsWritable` answers; `flattenRefusedBy` is nil unless the record
is a variant whose flatten is refused, and then it is the key of the base-chain record whose own refusal refuses
it (the refusal names that key; the store is unchanged). For an unmarked record `explained` through `occurrences`
are nil, otherwise they are the mark: whether it is explained, the first type read with extra slots, the slots it carried,
the slots the reader declares, and how many objects carried extra slots. A depth-0 type is the reader's document
type (the reference's names: `SurfaceDeclaring` for the consent store). For each row, open a copy of the store,
assert the mark and `IsWritable`, flatten the key when `flattenRefusedBy` is set and assert the refusal, then
remove the key: refused (with the mark, the file unchanged) exactly where
`writable` is false. In the consent store a write of `consented` lands and a write of `over-consent` is refused.
The carried-verbatim check: open `extra-slot-store.cc`, assert `marked` read-only and `clean` writable, write
`clean` and flush: `marked`'s payload bytes, schema id and stored-at, and every catalog entry's bytes, equal the
originals.
## Bounds (`bounds-outcomes.msgpack`)

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -7,6 +7,15 @@ All notable changes to this package are documented in this file.
### Breaking
- A single-file store holding two records under one key is unreadable: the reader refuses it with `CultStoreUnreadableException` naming the key (keys compare by ordinal code units, no case folding or normalisation), so a cache, `ReadStoreSnapshot` and a flush refuse the file and leave it as it is. Before, the later record silently replaced the earlier. A directory store reads its manifest through the same reader and refuses a duplicate key the same way. Repair the duplicate with the runtime that wrote it before reading the store here.
- A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable.
- `Flatten` (and `FlattenAsync`) of a variant whose base chain holds a record its store does not own (a record held read-only for extra slots, or undeclared drift that drops slots) is refused with `CultSchemaConflictException` naming that base record, and the store is left as it was. Before, the flatten wrote the base's decoded view into a plain record and the base's unread data was lost for that variant. Writing and removing the variant itself are still accepted: they persist only its delta.
- `MintElementIds` refuses a home store's commit while that store holds a record read-only for extra slots whose element ids exist only in memory, until the build that declares those slots rewrites it; commits to earlier stores stand.
- A type's declared slots are the array its own formatter writes for one instance, not its public `[Key]` members, so a non-public `[Key]` of an `AllowPrivate` type counts. Only a `[MessagePackObject]` type MessagePack writes by its `[Key]` layout is checked for extra slots. A type with another formatter (its own `[MessagePackFormatter]`, a formatter of its own from a `CultCacheFormatterResolver`, or the collection formatter of a `[MessagePackObject]` collection type) owns its layout and is not checked; a resolver that hands out MessagePack's own `[Key]`-layout formatter does not exempt the type; a type no instance of which can be written is not checked either (no parameterless constructor and a `[Key]` getter that throws on an uninitialized instance; ruled `cultlib-gaps:ruling:slot-unmeasurable-type`). Extra slots in those are read past and dropped on the next write, as before. Measuring runs the type's `[Key]` getters, and on failure its parameterless constructor, once per type per process on the first store read of it; a failure leaves the type unchecked.
### Added
- `CultCache.IsWritable(CultRecordKey)`: whether a removal of the key, or a write of the record's own type, would be let through, answered by the write's own decision: false for a foreign record, undeclared drift that drops slots, unread slots, a read-only home store, or a record no attached store is home to; true for a key the cache does not hold unless a store lists it foreign. It does not predict a `Flatten` refused for its base chain. Live and advisory: the write still decides.
- `CultUnreadSlots` (`TypeName`, `SlotCount`, `DeclaredCount`, `Occurrences`, `Explained`), `CultStoredDocument.UnreadSlots`, and `CultSchemaConflictException.UnreadSlots` with its constructor overload: what a load found past the reader's slots, holding no slot value. The refusal message names the type and both counts, never a value.
## [3.0.0] - 2026-11-11