C#: a whole-view write lays back a record under an id its type does not own

C2a F2, widened by Self: records and catalog entries a runtime does not own
survive its writes byte for byte. The schema-identity probe showed a C#
rewrite restamping every other runtime's record to its sha256 id and
dropping their catalog entries.

- The single-file store remembers, per key, a record it loaded under an id
  its type neither owns nor declares compatible (resolved through the
  catalog's schema name or an entry pointing at a local id), with the entry
  that published it. While the store still holds the entry it loaded, a
  whole-view write (flush or unconditional commit) lays that record back
  with its id, storedAt, bytes and catalog entry. A write of the record
  itself still stores it under the type's own id.
- A record published by an entry under the type's own id is this schema's
  and is re-encoded as before (the registered entry wins its own id).
- A laid-back record's load-minted element ids stay in memory only, so
  MintElementIds still rewrites it.
- The rolling-deploy tests lose their premise: an older version's whole
  view no longer sheds a newer record's member. They now pin that it lays
  the record back and the newer cache's condition holds; the A-B-A test
  keeps F6's defence through an explicit older write.
- Contract: the stamp and the lay-back rule in the persistence format.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-30 21:20:11 +02:00
commit a11d30fce2
5 changed files with 198 additions and 47 deletions

View file

@ -124,8 +124,8 @@ the payload's first field says. The catalog is the store's own description of it
A writer keeps the same invariant: every record it writes carries a schema id that some catalog
entry it writes publishes, as that entry's id or as one of its compatible ids. A cache stamps a
record with its registered schema's id, whatever id the record arrived under, when it loads the
record and when it puts one. The cache is the only owner of that stamp: a store writer never
record it puts with its registered schema's id, and a record it loads under an id its type owns or
declares compatible (see below for one it does not). The cache is the only owner of that stamp: a store writer never
changes the identity of a record it did not receive from its caller, so a record another writer
changed since this cache read it is written back as that writer left it.
@ -142,6 +142,15 @@ names a local type only when no local type has the id, for a store written by a
schema ids this one cannot know (a Rust type's schema id is its entry type). A schema renamed under
a stable id therefore opens in every runtime that holds the id.
Records and catalog entries a runtime does not own survive its writes. A record a cache resolved
through the catalog's schema name, or through an entry pointing at a local id, is readable, but its
id is not one its type owns or declares compatible: a write of another record, a whole-view write
included, lays it back with its id, storedAt, bytes and catalog entry exactly as they were stored.
Only a write of that record itself stores it under the type's own id. A writer that describes
schema ids itself (Rust, whose catalog entries are derived from its entry types) lays back the entry
the file holds for an id it rewrites, and describes an id only when the file has no entry for it or
when its registered type owns the id under another name (a rename).
A record that resolves to no local type is a foreign record, for example one of a type this build does not
have, or one renamed without declaring its old id. A store never destroys or relabels it: the store carries it
byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords`

View file

@ -3660,8 +3660,36 @@ namespace GameCult.Caching
Entries[stored.Key.Value] = stored;
SetLastSchemaMigrationReports(reports);
SetForeignRecords(foreign);
_unowned.Clear();
foreach (var record in snapshot.Records)
{
if (!Entries.TryGetValue(record.Key, out var held) || Owns(held.Descriptor, record.SchemaId))
continue;
// An entry under the type's own id that lists the record's id says the record is this schema's: it is re-encoded.
var publisher = Publisher(record.SchemaId, snapshot.SchemaCatalog);
if (!Owns(held.Descriptor, publisher.SchemaId))
_unowned[record.Key] = (held, record, publisher);
}
}
// The records this store loaded under an id their type neither owns nor lists as compatible (another runtime's id, or an
// earlier declaration of the schema, resolved through the catalog's name or its pointer to a local id), each with the
// catalog entry that published it, keyed while the store holds the entry it loaded. A whole-view write lays such a record
// back exactly as it was stored, id and catalog entry included; only a write of the record itself stores it under its
// type's own id.
private readonly Dictionary<string, (CultStoredDocument Loaded, CultPersistedRecord Record, CultSchemaCatalogEntry Entry)> _unowned =
new(StringComparer.Ordinal);
private static bool Owns(CultDocumentDescriptor descriptor, string schemaId) =>
string.Equals(descriptor.SchemaId, schemaId, StringComparison.Ordinal) ||
descriptor.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal);
// The entry that publishes a schema id: the one that owns it, else the first that lists it.
private static CultSchemaCatalogEntry Publisher(string schemaId, IReadOnlyCollection<CultSchemaCatalogEntry> catalog) =>
catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))
?? catalog.First(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal));
public override void Push(CultStoredDocument entry)
{
ThrowIfReadOnly();
@ -3695,15 +3723,15 @@ namespace GameCult.Caching
{
var disk = ReadSnapshot();
var foreign = ForeignOnDisk(disk);
var records = WholeView(Entries.Values, disk, foreign);
var view = WholeView(Entries.Values, disk, foreign);
WriteSnapshot(
records,
view.Records,
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(),
disk?.SchemaCatalog ?? Array.Empty<CultSchemaCatalogEntry>(),
view.Arrived,
Entries.Values.Any(entry => entry.HoldsIds),
disk?.FormatVersion,
wholeStore: foreign.Count == 0);
WroteWholeView(records);
WroteWholeView(view.Records, view.LaidBack);
}
MarkFlushSucceeded();
@ -3736,7 +3764,8 @@ namespace GameCult.Caching
throw Overwrites(carried.Foreign);
}
var records = (ontoDisk ? disk.Records : WholeView(Entries.Values, disk, foreign)).ToDictionary(record => record.Key, StringComparer.Ordinal);
var view = ontoDisk ? null : WholeView(Entries.Values, disk, foreign);
var records = (view?.Records ?? disk.Records).ToDictionary(record => record.Key, StringComparer.Ordinal);
// The committer judged the batch against its own view; the file may have moved since. A merge is judged on the
// set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving),
// what they removed (departing), and this batch.
@ -3756,24 +3785,36 @@ namespace GameCult.Caching
var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())
.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry()))
.ToArray();
WriteSnapshot(records.Values, registered, disk.SchemaCatalog, holdsIds, disk.FormatVersion, wholeStore: !ontoDisk && foreign.Count == 0);
WriteSnapshot(records.Values, registered, view?.Arrived ?? disk.SchemaCatalog, holdsIds, disk.FormatVersion, wholeStore: !ontoDisk && foreign.Count == 0);
foreach (var entry in request.Deletes)
Entries.TryRemove(entry.Key.Value, out _);
foreach (var entry in request.Upserts)
Entries[entry.Key.Value] = entry;
if (!ontoDisk)
WroteWholeView(records.Values);
if (view != null)
{
view.LaidBack.ExceptWith(replaced);
WroteWholeView(records.Values, view.LaidBack);
}
MarkFlushSucceeded();
return CultCommitOutcome.Committed;
}
// A record's (schema id, storedAt) names its stored bytes, and a commit condition trusts that name. So a whole-view write
// that stores a record differently is a new store of it and mints it a later storedAt: under another id than it was loaded
// under (a type re-encoding a record it read through an older or foreign schema, shedding what it lacks), or with other
// under (a type re-encoding a record it read under an older id it declares), or with other
// bytes than the file holds at that storedAt. A record written exactly as it is stored keeps its storedAt.
// The whole view also lays back every foreign record the file holds now, as the file holds it, and refuses an entry whose key
// the file holds as a foreign record.
private CultPersistedRecord[] WholeView(
// Records and catalog entries this runtime does not own survive the write: a record loaded under an id its type does not own
// and that nothing has written since is laid back as it was loaded, with the catalog entry that published it, and every
// foreign record the file holds now is laid back as the file holds it. An entry whose key the file holds as a foreign record
// is refused. Arrived is the catalog the write publishes those records by.
private sealed class WholeViewWrite
{
public CultPersistedRecord[] Records = Array.Empty<CultPersistedRecord>();
public CultSchemaCatalogEntry[] Arrived = Array.Empty<CultSchemaCatalogEntry>();
public HashSet<string> LaidBack = new(StringComparer.Ordinal);
}
private WholeViewWrite WholeView(
IEnumerable<CultStoredDocument> view,
CultPersistedStoreSnapshot? disk,
IReadOnlyDictionary<string, (CultForeignRecord Foreign, CultPersistedRecord Record)> foreign)
@ -3785,9 +3826,18 @@ namespace GameCult.Caching
throw Overwrites(carried.Foreign);
}
var write = new WholeViewWrite();
var laidBackEntries = new List<CultSchemaCatalogEntry>();
var onDisk = (disk?.Records ?? Array.Empty<CultPersistedRecord>()).ToDictionary(record => record.Key, StringComparer.Ordinal);
return entries.Select(entry =>
write.Records = entries.Select(entry =>
{
if (_unowned.TryGetValue(entry.Key.Value, out var loaded) && ReferenceEquals(loaded.Loaded, entry))
{
write.LaidBack.Add(entry.Key.Value);
laidBackEntries.Add(loaded.Entry);
return loaded.Record;
}
var record = ToPersistedRecord(entry, SerializePayload);
var unchanged = onDisk.TryGetValue(record.Key, out var stored) && stored.StoredAt == record.StoredAt
? SameStoredBytes(stored, record)
@ -3796,6 +3846,13 @@ namespace GameCult.Caching
record.StoredAt = CultCache.MintStoredAt(record.StoredAt);
return record;
}).Concat(foreign.Values.Select(carried => carried.Record)).ToArray();
// An entry that published a laid-back record is written as it was loaded, whatever the file's entry for that id says now.
var laidBackIds = laidBackEntries.Select(entry => entry.SchemaId).ToHashSet(StringComparer.Ordinal);
write.Arrived = laidBackEntries.Distinct()
.Concat((disk?.SchemaCatalog ?? Array.Empty<CultSchemaCatalogEntry>()).Where(entry => !laidBackIds.Contains(entry.SchemaId)))
.ToArray();
return write;
}
// The records the file holds that no registered type claims, by key.
@ -3824,9 +3881,10 @@ namespace GameCult.Caching
a.Path.Select(step => (step.Slot, step.ElementId)).SequenceEqual(b.Path.Select(step => (step.Slot, step.ElementId))))
.All(same => same));
// The file now holds every record written, under its type's own id at the storedAt the write gave it: the store's entries
// and the cache's say so.
private void WroteWholeView(IReadOnlyCollection<CultPersistedRecord> written)
// The file now holds every record written, under the id and at the storedAt the write gave it: the store's entries and the
// cache's say so. A laid-back record is on disk as it was loaded, so the cache learns nothing new about it: in particular the
// ids it minted at load are still in memory only.
private void WroteWholeView(IReadOnlyCollection<CultPersistedRecord> written, HashSet<string> laidBack)
{
foreach (var record in written)
{
@ -3836,7 +3894,7 @@ namespace GameCult.Caching
entry.StoredAt = record.StoredAt;
}
Cache?.WroteWholeView(written);
Cache?.WroteWholeView(written.Where(record => !laidBack.Contains(record.Key)));
}
// Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote

View file

@ -461,42 +461,42 @@ namespace GameCult.Caching.Tests
batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "derived-from-" + Extra(held))), RollY);
});
// v1 reads v2's record through the entry that lists v1's id, and its whole-view commit writes it back under v1's id without
// the member v1 lacks: a new store of k, under a later storedAt. A v2 cache that loaded the record before that holds a
// condition on bytes that are gone: the condition fails, and nothing derived from the lost member lands. Pulling then shows
// the record as v1 left it, and a condition on that holds.
[TestCase(false)]
[TestCase(true)]
public void AWholeViewRewriteByAnOlderVersionFailsANewerVersionsCondition(bool unchangedOnly)
// v1 reads v2's record through the entry that lists v1's id: an id v1 does not own. Its whole-view write lays k back exactly
// as v2 stored it, the member v1 lacks included, under v2's id and storedAt, so a v2 cache's condition on k still holds and
// what it derives from that member lands.
[Test]
public void AWholeViewWriteByAnOlderVersionLeavesANewerRecordAsStored([Values] bool unchangedOnly, [Values] bool viaFlush)
{
WriteImportant();
var written = Read(RollPath).Records.Single();
var written = Read(RollPath);
using var newer = OpenRoll(RollV2);
var current = newer.Get(RollK)!;
WholeViewWrite(RollV1, viaFlush: false);
WholeViewWrite(RollV1, viaFlush);
var rewritten = Read(RollPath).Records.Single();
Assert.That(rewritten.SchemaId, Is.EqualTo(RollV1Id), "v1 rewrote k under its id");
Assert.That(rewritten.Payload, Is.Not.EqualTo(written.Payload), "and without Extra");
Assert.That(string.CompareOrdinal(rewritten.StoredAt, written.StoredAt), Is.GreaterThan(0), "a new store of k, so a later storedAt");
var rewritten = Read(RollPath);
var record = rewritten.Records.Single();
Assert.That((record.SchemaId, record.StoredAt, record.Payload), Is.EqualTo((written.Records.Single().SchemaId, written.Records.Single().StoredAt, written.Records.Single().Payload)));
Assert.That(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = rewritten.SchemaCatalog }),
Is.EqualTo(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = written.SchemaCatalog })),
"v2's catalog entry is written as v2 wrote it");
Assert.That(Derive(newer, current, unchangedOnly), Is.EqualTo(CultCommitOutcome.Mismatch));
using (var reopened = OpenRoll(RollV2))
{
Assert.That(reopened.Get(RollY), Is.Null, "nothing derived from the shed member landed");
Assert.That(Extra(reopened.Get(RollK)), Is.Empty);
}
newer.PullAllBackingStoresAsync().GetAwaiter().GetResult();
var pulled = newer.Get(RollK)!;
Assert.That(Extra(pulled), Is.Empty, "a pull replaces the record v1 rewrote");
Assert.That(Derive(newer, pulled, unchangedOnly), Is.EqualTo(CultCommitOutcome.Committed));
Assert.That(Derive(newer, current, unchangedOnly), Is.EqualTo(CultCommitOutcome.Committed));
using var reopened = OpenRoll(RollV2);
Assert.That(Extra(reopened.Get(RollK)), Is.EqualTo("important"));
Assert.That(Extra(reopened.Get(RollY)), Is.EqualTo("derived-from-important"));
}
// A-B-A: v1 sheds Extra from k, then another v2 cache (a restart is enough) loads k under v1's id and writes its whole view,
// putting k back under v2's id: the id the first v2 cache loaded k under, with bytes that no longer hold Extra. That cache's
// condition must still fail, through commits and through plain flushes alike.
// v1 writes k itself: a write of the record, so it stores what v1 holds, under v1's id and without the member v1 lacks.
private void OlderWritesK()
{
using var older = OpenRoll(RollV1);
older.Commit(batch => batch.Upsert(RollV1, older.Get(RollK)!, RollK));
}
// A-B-A: v1 writes k and sheds Extra, then another v2 cache (a restart is enough) loads k under v1's id and writes its whole
// view, putting k back under v2's id: the id the first v2 cache loaded k under, with bytes that no longer hold Extra. That
// cache's condition must still fail, through commits and through plain flushes alike.
[TestCase(false, false)]
[TestCase(true, false)]
[TestCase(false, true)]
@ -508,7 +508,8 @@ namespace GameCult.Caching.Tests
using var held = OpenRoll(RollV2);
var current = held.Get(RollK)!;
WholeViewWrite(RollV1, viaFlush);
OlderWritesK();
Assert.That(Read(RollPath).Records.Single().SchemaId, Is.EqualTo(RollV1Id), "v1 wrote k under its id");
WholeViewWrite(RollV2, viaFlush);
var restored = Read(RollPath).Records.Single();
Assert.That(restored.SchemaId, Is.EqualTo(written.SchemaId), "the second v2 cache put k back under v2's id");
@ -526,7 +527,7 @@ namespace GameCult.Caching.Tests
public void AFlushRestampsTheRecordItReencodesInTheCacheThatFlushed()
{
WriteImportant();
WholeViewWrite(RollV1, viaFlush: true);
OlderWritesK();
var shed = Read(RollPath).Records.Single();
Assert.That(shed.SchemaId, Is.EqualTo(RollV1Id));

View file

@ -212,6 +212,31 @@ namespace GameCult.Caching.Tests
Assert.That(reader.MintElementIds(), Is.EqualTo(0));
}
// A record from before ids, under another runtime's id for the same schema, is laid back by a whole-view write exactly as it
// was stored. The ids it minted at load are still in memory only, so MintElementIds still rewrites it, under the type's own id.
[Test]
public void AWholeViewWriteLeavesTheLoadMintedIdsOfARecordItLaysBackInMemoryOnly()
{
var path = PathOf("unowned-pre-id.cc");
WritePreIdStore(path, null, ("old", "o", 1));
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var ownId = snapshot.Records.Single().SchemaId;
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = "other.runtime.deck";
entry.CompatibleSchemaIds = new[] { "other.runtime.deck" };
snapshot.Records.Single().SchemaId = "other.runtime.deck";
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
var stored = DiskRecord(path, "old");
using var cache = Open(path);
cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("other"), new CultRecordKey("other")));
var laidBack = DiskRecord(path, "old");
Assert.That((laidBack.SchemaId, laidBack.StoredAt, laidBack.Payload), Is.EqualTo((stored.SchemaId, stored.StoredAt, stored.Payload)));
Assert.That(cache.MintElementIds(), Is.EqualTo(1), "its ids are still in memory only");
Assert.That(DiskRecord(path, "old").SchemaId, Is.EqualTo(ownId), "a write of the record stores it under the type's own id");
}
// A whole-view write that persists the ids a record minted at load stores other bytes under the same id: a new store of
// the record, so it takes a later storedAt, and the cache that wrote it holds that storedAt. A record it stores exactly as
// the file holds it keeps its own.

View file

@ -280,6 +280,64 @@ namespace GameCult.Caching.Tests
Assert.That(Read(path).FormatVersion, Is.EqualTo(carriesForeign ? CultPersistedStoreSnapshot.FormatV3 : CultPersistedStoreSnapshot.FormatV1));
}
private const string OtherRuntimeId = "other.runtime.deck";
// A deck another runtime wrote under its own id for the deck's schema. This build reads it through the catalog's schema name,
// but does not own the id.
private string OtherRuntimeStore(string name)
{
var path = PathOf(name);
using (var cache = Open(path, DeckOnly, directory: false))
cache.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D));
var snapshot = Read(path);
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = OtherRuntimeId;
entry.ContentHash = OtherRuntimeId;
entry.CompatibleSchemaIds = new[] { OtherRuntimeId };
snapshot.Records.Single().SchemaId = OtherRuntimeId;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
return path;
}
private static byte[] CatalogEntryBytes(string path, string schemaId) =>
CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot
{
SchemaCatalog = Read(path).SchemaCatalog.Where(entry => entry.SchemaId == schemaId).ToArray()
});
// Records and catalog entries a runtime does not own survive its writes: a write of another record lays the record back with
// its id, storedAt, bytes and catalog entry as they were stored.
[Test]
public void AWriteOfAnotherRecordLaysBackARecordUnderAnIdThisBuildDoesNotOwn([Values] Write write)
{
var path = OtherRuntimeStore("other-runtime.cc");
var before = Stored(path, D);
var entry = CatalogEntryBytes(path, OtherRuntimeId);
using (var cache = Open(path, DeckOnly, directory: false))
{
Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d"), "this build reads it");
Land(cache, write, E, DeckOf("e"));
}
Assert.That(Stored(path, D), Is.EqualTo(before));
Assert.That(CatalogEntryBytes(path, OtherRuntimeId), Is.EqualTo(entry));
}
// A write of the record itself stores what the cache holds, under the type's own id.
[Test]
public void AWriteOfTheRecordItselfStoresItUnderTheTypesOwnId([Values(Write.Flush, Write.Commit)] Write write)
{
var path = OtherRuntimeStore("rewritten.cc");
using (var cache = Open(path, DeckOnly, directory: false))
Land(cache, write, D, DeckOf("changed"));
Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(DeckOnly.GetRequired(Deck).SchemaId));
using var reopened = Open(path, DeckOnly, directory: false);
Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("changed"));
}
// A commit onto the file that involves a variant is judged on the set the file will hold. A foreign record is not decoded
// for that judgement, so it neither fails the commit nor joins the set.
[Test]