C#: a document type declares the older schema ids its records may sit under

CultDocumentAttribute.CompatibleSchemaIds, as TypeScript and Python already name it. The
registered catalog entry lists the ids after its own, the registry resolves a record under one
to the type (its own id first, then the first registered type that declares it), and a
condition on a record compares schema ids through that resolution, so a record still under a
declared old id counts as the record the cache holds. The cache already stamps every record it
writes with the descriptor's own id; the store writers still change no identity, so a record a
commit does not rewrite stays under the id the file holds it under, published by the registered
entry. Without the declaration such a commit is refused, typed, as before.

Contract and XML docs describe the declaration.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-30 11:30:05 +02:00
commit 186e5b29ae
5 changed files with 329 additions and 11 deletions

View file

@ -178,7 +178,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
if (commitLease == null)
return CultCommitOutcome.Contended;
var manifest = ReadManifest();
if (!request.ConditionsHold(manifest.Records, Entries.Values))
if (!request.ConditionsHold(manifest.Records, Entries.Values, Registry.CanonicalSchemaId))
return CultCommitOutcome.Mismatch;
var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);

View file

@ -129,6 +129,13 @@ record and when it puts one. The cache is the only owner of that stamp: a store
changes the identity of a record it did not receive from its caller, so a record another writer
changed since this cache read it is written back as that writer left it.
A schema type declares the older ids its records may sit under: `CultDocumentAttribute.CompatibleSchemaIds`
(C#), `compatibleSchemaIds` (TypeScript, Python). The registered catalog entry lists them after its own id, and
the cache restamps a record loaded under one to the type's own id, so the next write of that record carries the
registered id. A record another writer left under a declared id, and that a commit does not rewrite, stays under it
and the registered entry publishes it. Without the declaration, a C# commit onto a file that holds a record under an
id only the file's entry lists is refused, typed, and the file is left as it was. Rust has no such declaration.
A cache resolves a record to a local type by the record's schema id: the registered type that owns
the id, else one that lists it as compatible. The name the catalog entry carries is metadata. It
names a local type only when no local type has the id, for a store written by a runtime whose
@ -151,9 +158,8 @@ an own id and disagree on the schema name, when records of different types share
write would change which type an existing record resolves to (the schema names of the records already
in the store and of the records written differ), or when a record's id is published by no chosen entry.
The retyping refusal has one exception, a rename: when the entries of both the existing records and
the written ones own the id, the registered descriptor wins. (A C# cache stamps a record with its
descriptor's own id, so it cannot write a record under an id its descriptor only lists, and the
retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the
the written ones own the id, the registered descriptor wins. (A C# cache stamps every record it
writes with its descriptor's own id, so the retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the
schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError`
(Rust, TypeScript, Python).

View file

@ -241,6 +241,7 @@ namespace GameCult.Caching
string schemaName,
string schemaVersion,
string schemaId,
IReadOnlyList<string> compatibleSchemaIds,
string contentHash,
string canonicalSchemaJson,
bool isGlobal,
@ -254,6 +255,7 @@ namespace GameCult.Caching
SchemaName = schemaName;
SchemaVersion = schemaVersion;
SchemaId = schemaId;
CompatibleSchemaIds = compatibleSchemaIds;
ContentHash = contentHash;
CanonicalSchemaJson = canonicalSchemaJson;
IsGlobal = isGlobal;
@ -279,6 +281,9 @@ namespace GameCult.Caching
public string SchemaName { get; }
public string SchemaVersion { get; }
public string SchemaId { get; }
/// <summary>The ids besides <see cref="SchemaId"/> this type declared through <see cref="CultDocumentAttribute.CompatibleSchemaIds"/>.</summary>
public IReadOnlyList<string> CompatibleSchemaIds { get; }
public string ContentHash { get; }
public string CanonicalSchemaJson { get; }
public bool IsGlobal { get; }
@ -370,7 +375,7 @@ namespace GameCult.Caching
SchemaVersion = SchemaVersion,
ContentHash = ContentHash,
CanonicalSchemaJson = CanonicalSchemaJson,
CompatibleSchemaIds = [SchemaId],
CompatibleSchemaIds = [SchemaId, .. CompatibleSchemaIds],
Members = Members
.OrderBy(member => member.Slot)
.Select(member => new CultSchemaMemberCatalogEntry
@ -578,6 +583,16 @@ namespace GameCult.Caching
throw new InvalidOperationException($"Unknown CultCache schema id '{schemaId}'.");
}
// The id a record under schemaId is stamped with: the own id of the type that owns it, else of the first registered
// type that declares it compatible; an id no registered type claims is its own.
internal string CanonicalSchemaId(string schemaId)
{
var indexes = _indexes;
return indexes.BySchemaId.TryGetValue(schemaId, out var owner) || indexes.ByCompatibleSchemaId.TryGetValue(schemaId, out owner)
? owner.SchemaId
: schemaId;
}
public CultDocumentDescriptor ResolvePersistedSchema(string schemaId, IReadOnlyCollection<CultSchemaCatalogEntry> catalog)
{
return ResolvePersistedSchemaDetailed(schemaId, catalog).Descriptor;
@ -613,6 +628,11 @@ namespace GameCult.Caching
throw new InvalidOperationException($"Persisted schema '{schemaId}' is not present in the embedded catalog.");
}
if (indexes.ByCompatibleSchemaId.TryGetValue(schemaId, out var declared))
{
return BuildCompatibleResolutionResult(persisted, declared);
}
foreach (var compatibleSchemaId in persisted.CompatibleSchemaIds.Where(candidate => !string.IsNullOrWhiteSpace(candidate)))
{
if (indexes.BySchemaId.TryGetValue(compatibleSchemaId, out var compatibleLocal))
@ -687,6 +707,8 @@ namespace GameCult.Caching
indexes.ByType[descriptor.DocumentType] = descriptor;
indexes.BySchemaId[descriptor.SchemaId] = descriptor;
foreach (var compatibleSchemaId in descriptor.CompatibleSchemaIds)
indexes.ByCompatibleSchemaId.TryAdd(compatibleSchemaId, descriptor);
indexes.BySchemaName[descriptor.SchemaName] = schemaNameVersions == null
? [descriptor]
: [.. schemaNameVersions, descriptor];
@ -717,6 +739,7 @@ namespace GameCult.Caching
{
ByType = new Dictionary<Type, CultDocumentDescriptor>();
BySchemaId = new Dictionary<string, CultDocumentDescriptor>(StringComparer.Ordinal);
ByCompatibleSchemaId = new Dictionary<string, CultDocumentDescriptor>(StringComparer.Ordinal);
BySchemaName = new Dictionary<string, CultDocumentDescriptor[]>(StringComparer.Ordinal);
}
@ -724,11 +747,13 @@ namespace GameCult.Caching
{
ByType = new Dictionary<Type, CultDocumentDescriptor>(source.ByType);
BySchemaId = new Dictionary<string, CultDocumentDescriptor>(source.BySchemaId, StringComparer.Ordinal);
ByCompatibleSchemaId = new Dictionary<string, CultDocumentDescriptor>(source.ByCompatibleSchemaId, StringComparer.Ordinal);
BySchemaName = new Dictionary<string, CultDocumentDescriptor[]>(source.BySchemaName, StringComparer.Ordinal);
}
public Dictionary<Type, CultDocumentDescriptor> ByType { get; }
public Dictionary<string, CultDocumentDescriptor> BySchemaId { get; }
public Dictionary<string, CultDocumentDescriptor> ByCompatibleSchemaId { get; }
public Dictionary<string, CultDocumentDescriptor[]> BySchemaName { get; }
}
@ -765,12 +790,19 @@ namespace GameCult.Caching
var contentHash = Sha256(schemaJson);
var semanticFingerprint = BuildSemanticFingerprint(attribute.SchemaName, attribute.SchemaVersion, descriptorMembers);
var schemaId = Sha256(semanticFingerprint);
if (attribute.CompatibleSchemaIds.Any(string.IsNullOrWhiteSpace))
throw new InvalidOperationException($"Type {type.FullName} declares an empty compatible schema id.");
var compatibleSchemaIds = attribute.CompatibleSchemaIds
.Where(id => !string.Equals(id, schemaId, StringComparison.Ordinal))
.Distinct(StringComparer.Ordinal)
.ToArray();
return new CultDocumentDescriptor(
type,
attribute.SchemaName,
attribute.SchemaVersion,
schemaId,
compatibleSchemaIds,
contentHash,
schemaJson,
type.GetCustomAttribute<CultGlobalAttribute>() != null,
@ -1783,7 +1815,7 @@ namespace GameCult.Caching
// durable is what the store holds on disk now; observed is what its cache last loaded or committed.
// Identity is (schemaId, storedAt), sound because every write to a key mints a later storedAt.
public bool ConditionsHold(IReadOnlyCollection<CultPersistedRecord> durable, IEnumerable<CultStoredDocument> observed)
public bool ConditionsHold(IReadOnlyCollection<CultPersistedRecord> durable, IEnumerable<CultStoredDocument> observed, Func<string, string> canonicalSchemaId)
{
var byKey = durable.ToDictionary(record => record.Key, StringComparer.Ordinal);
foreach (var (key, schemaId, storedAt) in Expected)
@ -1791,13 +1823,13 @@ namespace GameCult.Caching
byKey.TryGetValue(key.Value, out var record);
var holds = schemaId == null
? record == null
: record != null && record.SchemaId == schemaId && record.StoredAt == storedAt;
: record != null && canonicalSchemaId(record.SchemaId) == schemaId && record.StoredAt == storedAt;
if (!holds)
return false;
}
return !ExpectUnchanged ||
durable.Select(record => Identity(record.Key, record.SchemaId, record.StoredAt))
durable.Select(record => Identity(record.Key, canonicalSchemaId(record.SchemaId), record.StoredAt))
.OrderBy(identity => identity, StringComparer.Ordinal)
.SequenceEqual(observed
.Select(stored => Identity(stored.Key.Value, stored.Descriptor.SchemaId, stored.StoredAt))
@ -2542,7 +2574,7 @@ namespace GameCult.Caching
var inMemory = _entries.Values
.Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt })
.ToArray();
return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch;
return request.ConditionsHold(inMemory, _entries.Values, Registry.CanonicalSchemaId) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch;
});
});
}
@ -3607,7 +3639,7 @@ namespace GameCult.Caching
return CultCommitOutcome.Contended;
var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot();
if (!request.ConditionsHold(disk.Records, Entries.Values))
if (!request.ConditionsHold(disk.Records, Entries.Values, Registry.CanonicalSchemaId))
return CultCommitOutcome.Mismatch;
// An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins,
@ -3664,7 +3696,7 @@ namespace GameCult.Caching
{
if (landing.Contains(record.Key) || removing.Contains(record.Key))
continue;
if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == record.SchemaId)
if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.Descriptor.SchemaId == Registry.CanonicalSchemaId(record.SchemaId))
continue;
arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _));
}

View file

@ -18,6 +18,14 @@ namespace GameCult.Caching
public string SchemaName { get; }
public string SchemaVersion { get; }
/// <summary>
/// Schema ids other than this type's own that a store may hold its records under, typically the ids an earlier
/// version of the schema had. The registered catalog entry lists them, and a cache resolves a record stored under
/// one of them to this type (its own id is tried first) and restamps it to the type's own id on load. TypeScript
/// and Python name the same declaration <c>compatibleSchemaIds</c>.
/// </summary>
public string[] CompatibleSchemaIds { get; set; } = Array.Empty<string>();
}
[AttributeUsage(AttributeTargets.Class, Inherited = false)]

View file

@ -0,0 +1,272 @@
#nullable enable
using System;
using System.IO;
using System.Linq;
using GameCult.Caching.MessagePack;
using MessagePack;
using NUnit.Framework;
namespace GameCult.Caching.Tests
{
// A document type declares the older schema ids its records may sit under (CultDocumentAttribute.CompatibleSchemaIds). The
// registered catalog entry lists them, a cache resolves a record under one to that type and stamps it with the type's own id,
// and a store writer never changes the identity of a record it did not receive.
public class CompatibleSchemaIdTests
{
private const string OldId = "old.id";
private const string OwnerId = "sha256:86186abf7d63d8a326c725d39a10f3a4db11b1e8698e8c0fcf6d9720031bc480";
private const string SharedId = "shared.id";
[CultDocument("tests.compat_deck", "tests.compat_deck.v2", CompatibleSchemaIds = new[] { OldId })]
[MessagePackObject]
public sealed class DeclaringDeck
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
// The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing.
[CultDocument("tests.compat_deck", "tests.compat_deck.v2")]
[MessagePackObject]
public sealed class BareDeck
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
[CultDocument("tests.compat_owner", "tests.compat_owner.v1", CompatibleSchemaIds = new[] { OwnerId })]
[MessagePackObject]
public sealed class Owner
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
[CultDocument("tests.compat_claimant", "tests.compat_claimant.v1", CompatibleSchemaIds = new[] { OwnerId, SharedId })]
[MessagePackObject]
public sealed class Claimant
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
[CultDocument("tests.compat_second_claimant", "tests.compat_second_claimant.v1", CompatibleSchemaIds = new[] { SharedId })]
[MessagePackObject]
public sealed class SecondClaimant
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
[CultDocument("tests.compat_empty", "tests.compat_empty.v1", CompatibleSchemaIds = new[] { " " })]
[MessagePackObject]
public sealed class EmptyClaim
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
private static readonly CultDocumentRegistry Declaring = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaringDeck) });
private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { typeof(BareDeck) });
private static readonly CultRecordKey D = new("d");
private string _directory = "";
[SetUp]
public void SetUp()
{
_directory = Path.Combine(Path.GetTempPath(), "cultlib-compat-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(_directory);
}
[TearDown]
public void TearDown()
{
if (Directory.Exists(_directory))
Directory.Delete(_directory, true);
}
private static CultCache Open(string path, CultDocumentRegistry registry) =>
CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
private static CultPersistedStoreSnapshot Read(string path) =>
CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
// A store written by a runtime that knew the schema under an older id and another name: one entry that owns OldId, one
// record under it. Nothing but the declaration lets this runtime's type claim it.
private string OldStore(string name)
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D));
var snapshot = Read(path);
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = OldId;
entry.SchemaName = "tests.legacy_deck";
entry.CompatibleSchemaIds = new[] { OldId };
snapshot.Records.Single().SchemaId = OldId;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
return path;
}
// The same store with the entry keeping this schema's name and only listing the older id: what a store carries after a
// writer that did not declare it, so a runtime without the declaration can still read it.
private string OldStoreUnderTheSameName(string name)
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(typeof(BareDeck), new BareDeck { Name = "d" }, D));
var snapshot = Read(path);
var entry = snapshot.SchemaCatalog.Single();
entry.ContentHash = "stale";
entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId };
snapshot.Records.Single().SchemaId = OldId;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
return path;
}
[Test]
public void ARecordUnderADeclaredOldIdOpensAndACommitRewritesItUnderTheRegisteredId()
{
var path = OldStore("old.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
using (var cache = Open(path, Declaring))
{
Assert.That(cache.Get<DeclaringDeck>(D)!.Name, Is.EqualTo("d"), "the declaration resolves the record to the type");
Assert.That(cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d2" }, D)), Is.True);
}
var written = Read(path);
Assert.That(written.Records.Single().SchemaId, Is.EqualTo(descriptor.SchemaId));
var entry = written.SchemaCatalog.Single();
Assert.That(entry.SchemaId, Is.EqualTo(descriptor.SchemaId));
Assert.That(entry.SchemaName, Is.EqualTo("tests.compat_deck"));
Assert.That(entry.CompatibleSchemaIds, Is.EqualTo(new[] { descriptor.SchemaId, OldId }));
foreach (var registry in new[] { Declaring, Bare })
{
using var reopened = Open(path, registry);
Assert.That(reopened.AllEntries.Count(), Is.EqualTo(1));
}
}
[Test]
public void ARecordUnderAnOldIdNoTypeDeclaresIsNotOpenedByThatSchemaName()
{
var path = OldStore("undeclared.cc");
Assert.That(() => Open(path, Bare), Throws.InstanceOf<Exception>(), "the entry's name is metadata; only a declared id resolves it");
}
// An unconditional commit writes the cache's whole view: the record the batch does not name is stamped with the registered
// id too, because the cache stamped it when it loaded it.
[Test]
public void ACommitThatWritesTheWholeViewStampsEveryLoadedRecordWithTheRegisteredId()
{
var path = OldStore("whole-view.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
using (var cache = Open(path, Declaring))
cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")));
var written = Read(path);
Assert.That(written.Records.Select(record => record.SchemaId), Is.All.EqualTo(descriptor.SchemaId));
Assert.That(written.Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e" }));
}
// A commit onto the file writes its batch and leaves every other record as the file holds it: the store writer never
// changes identity. The record stays under the older id, which the registered entry publishes as compatible.
[Test]
public void ACommitOntoTheFileLeavesARecordUnderADeclaredOldIdAsItIsAndPublishesTheId()
{
var path = OldStoreUnderTheSameName("onto.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
using (var cache = Open(path, Declaring))
{
Assert.That(cache.Get<DeclaringDeck>(D), Is.Not.Null);
Assert.That(cache.Commit(batch =>
{
batch.Expect(new CultRecordKey("e"), null);
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.True);
}
var written = Read(path);
Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId));
Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId));
Assert.That(written.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain(OldId));
using var reopened = Open(path, Declaring);
Assert.That(reopened.Get<DeclaringDeck>(D), Is.Not.Null);
}
[Test]
public void WithoutTheDeclarationACommitOntoTheFileIsRefusedTypedAndTheFileLeftAsItWas()
{
var path = OldStoreUnderTheSameName("undeclared-onto.cc");
var bytes = File.ReadAllBytes(path);
using var cache = Open(path, Bare);
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch =>
{
batch.Expect(new CultRecordKey("e"), null);
batch.Upsert(typeof(BareDeck), new BareDeck { Name = "e" }, new CultRecordKey("e"));
}))!;
Assert.That(refusal.SchemaId, Is.EqualTo(OldId));
Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes));
}
// A record under a declared old id is the same record the cache holds under the registered id: a condition naming it holds.
[Test]
public void AConditionOnARecordUnderADeclaredOldIdHoldsAndTheWholeStoreCountsAsUnchanged()
{
var path = OldStoreUnderTheSameName("conditions.cc");
using var cache = Open(path, Declaring);
var current = cache.Get<DeclaringDeck>(D)!;
Assert.That(cache.TryCommit(batch =>
{
batch.Expect(D, current);
batch.ExpectUnchanged();
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.EqualTo(CultCommitOutcome.Committed));
}
[Test]
public void ADeclarationIsRegisteredWithoutItsOwnIdAndItsEntryListsTheOwnIdFirst()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Owner) });
var descriptor = registry.GetRequired<Owner>();
Assert.That(descriptor.SchemaId, Is.EqualTo(OwnerId), "the pinned id is this schema's id");
Assert.That(descriptor.CompatibleSchemaIds, Is.Empty, "a type's own id is not a declared compatible id");
Assert.That(descriptor.ToCatalogEntry().CompatibleSchemaIds, Is.EqualTo(new[] { OwnerId }));
}
[Test]
public void AnIdATypeOwnsResolvesToItEvenWhenAnEarlierTypeDeclaresItCompatible()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(Owner) });
var owner = registry.GetRequired<Owner>();
var entry = owner.ToCatalogEntry();
Assert.That(registry.CanonicalSchemaId(OwnerId), Is.EqualTo(OwnerId));
Assert.That(registry.ResolvePersistedSchema(OwnerId, new[] { entry }), Is.SameAs(owner));
}
[Test]
public void AnIdTwoTypesDeclareCompatibleResolvesToTheOneRegisteredFirst()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Claimant), typeof(SecondClaimant) });
var first = registry.GetRequired<Claimant>();
Assert.That(registry.CanonicalSchemaId(SharedId), Is.EqualTo(first.SchemaId));
}
[Test]
public void AnIdNoTypeClaimsIsItsOwnCanonicalId()
{
Assert.That(Declaring.CanonicalSchemaId("nobody.id"), Is.EqualTo("nobody.id"));
}
[Test]
public void AnEmptyDeclaredIdIsRefused()
{
Assert.That(() => CultDocumentRegistry.ForTypes(new[] { typeof(EmptyClaim) }).GetRequired<EmptyClaim>(),
Throws.InvalidOperationException.With.Message.Contains("empty compatible schema id"));
}
}
}