GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Apply refusal: a write over a durable record under an undeclared id is refused at ApplyWriteSet and reloads read-only

Browse source 
ForeignOnDisk and the two RefuseForeign calls in PushAll and CommitBatchCore are
gone. ApplyWriteSet refuses first, over every key it writes or removes, by
Registry.Declares for a cache and by the catalog entry it writes for a cacheless
CultMesh write. Both refusals reload through one DurableAt; the directory store
reloads through LoadPage, the per-record body of LoadRecordPages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-03 13:08:44 +02:00
commit 067df993a3
4 changed files with 123 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -185,7 +185,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values))
return CultCommitOutcome.Mismatch;
RefuseForeign(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value));
RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value));
var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
var previousDirtyKeys = _dirtyKeys.Keys.ToArray();
var previousDeletedKeys = _deletedKeys.Keys.ToArray();
@ -236,18 +236,23 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Directory.CreateDirectory(_manifestFile.DirectoryName!);
using var commitLease = AcquireCommitLease(wait: true, create: true);
var (manifest, catalogBytes) = ReadManifest();
RefuseForeign(manifest, Array.Empty<string>());
RefuseUndeclaredOnDisk(manifest, Array.Empty<string>());
WriteGeneration(manifest, catalogBytes);
});
}
// The manifest's other records stay as they are, foreign ones included; a write that would replace or remove a foreign
// record is refused before any page is written, by the one rule every store shares (CacheBackingStore.RefuseForeign).
private void RefuseForeign(CultPersistedStoreSnapshot currentManifest, IEnumerable<string> written) =>
RefuseForeign(
// The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a
// record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every
// store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them.
private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable<string> written)
{
CultPersistedRecord? HeldAt(string key) =>
currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
RefuseUndeclaredOnDisk(
written.Concat(_deletedKeys.Keys).Concat(_dirtyKeys.Keys),
key => currentManifest.Records.FirstOrDefault(record => record.Key == key) is { } held
? Foreign(held, currentManifest.SchemaCatalog)
key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Declares) : null,
key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog) == null
? LoadPage(held, currentManifest.SchemaCatalog, out _, out _)
: null,
key =>
{
@ -255,6 +260,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
_deletedKeys.TryRemove(key, out _);
IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty;
});
}
// Runs under the commit lease: pages first, then the manifest that names them.
private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary<CultSchemaCatalogEntry, byte[]> catalogBytes)
@ -383,20 +389,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
index =>
{
var started = tracePages ? Stopwatch.GetTimestamp() : 0L;
var metadata = records[index];
var record = ReadPersistedRecordPage(metadata, out var pagePayload);
var stored = ToStoredDocument(
record,
VersionOf(metadata.Payload),
catalogEntries,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
out recordReports[index]);
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
stored.StoredSchemaId = metadata.SchemaId;
storedRecords[index] = stored;
storedRecords[index] = LoadPage(records[index], catalogEntries, out recordReports[index], out var pageLength);
if (tracePages)
{
pageBytes[index] = pagePayload.LongLength;
pageBytes[index] = pageLength;
pageElapsedTicks[index] = Stopwatch.GetTimestamp() - started;
}
});
@ -557,6 +553,26 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
}
// One record as a load decodes it, from the page its manifest record names: a pull's pages and a refusal's reload.
private CultStoredDocument LoadPage(
CultPersistedRecord metadata,
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
out CultSchemaMigrationReport report,
out long pageLength)
{
var record = ReadPersistedRecordPage(metadata, out var pagePayload);
pageLength = pagePayload.LongLength;
var stored = ToStoredDocument(
record,
VersionOf(metadata.Payload),
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
out report);
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
stored.StoredSchemaId = metadata.SchemaId;
return stored;
}
private CultPersistedRecord ReadPersistedRecordPage(CultPersistedRecord metadata, out byte[] pagePayload)
{
var path = ContentAddressedRecordPath(metadata);

114
src/GameCult.Caching/CultCache.cs
View file

@ -693,9 +693,7 @@ namespace GameCult.Caching
if (indexes.BySchemaId.TryGetValue(schemaId, out var exact))
return exact;
// A schema is published by its id or as a compatible id.
persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))
?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal));
persisted = PublishedBy(schemaId, catalog);
if (persisted == null)
{
throw new InvalidOperationException($"Persisted schema '{schemaId}' is not present in the embedded catalog.");
@ -735,6 +733,11 @@ namespace GameCult.Caching
return null;
}
// The catalog entry a record under this id is published by: the entry that owns the id, else the first that lists it compatible.
internal static CultSchemaCatalogEntry? PublishedBy(string schemaId, IReadOnlyCollection<CultSchemaCatalogEntry> catalog) =>
catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))
?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal));
// The one answer to whether a cache may write, relabel or remove a record stored under this id: a registered type owns it or
// declares it compatible. Read live, so a type registered later that declares the id claims its records at their next write.
internal bool Declares(string schemaId)
@ -3632,21 +3635,32 @@ namespace GameCult.Caching
throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) });
}
// The one decision of whether a write may proceed over what the file holds as foreign, for every store kind. A key this
// cache writes or removes (staged, or in a batch) that the file now holds foreign is refused, every such key at once,
// through Refuse: a foreign record is not held, so the cache drops its copy and the store forgets its staged change. A record
// this cache holds but did not write is not refused: it stays until a pull drops it, and the write that never touched it lands.
protected void RefuseForeign(IEnumerable<string> written, Func<string, CultForeignRecord?> foreignAt, Action<string> unstage)
// A stored record under an id the writer does not declare, named by the catalog entry that publishes the id. Header only: no
// payload is decoded. Null for a record under a declared id.
protected static (string Key, string SchemaId, string SchemaName)? Undeclared(
CultPersistedRecord record, IReadOnlyCollection<CultSchemaCatalogEntry> catalog, Func<string, bool> declares) =>
declares(record.SchemaId)
? null
: (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty);
// The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. A key this write names
// whose durable record is under an id no registered type declares is refused, every such key at once, through Refuse, which
// reloads it as the store holds it: dropped when foreign, held read-only when a type reads it.
protected void RefuseUndeclaredOnDisk(
IEnumerable<string> written,
Func<string, (string Key, string SchemaId, string SchemaName)?> undeclaredAt,
Func<string, CultStoredDocument?> durableAt,
Action<string> unstage,
bool cacheless = false)
{
var refused = written.Distinct(StringComparer.Ordinal)
.Select(foreignAt)
.OfType<CultForeignRecord>()
.Select(undeclaredAt)
.OfType<(string Key, string SchemaId, string SchemaName)>()
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray();
if (refused.Length == 0)
return;
Refuse(refused.Select(record => record.Key), _ => null, unstage,
inner => Overwrites(inner, refused.Select(record => (record.Key, record.SchemaId, record.SchemaName)).ToArray()));
Refuse(refused.Select(record => record.Key), durableAt, unstage, inner => Overwrites(inner, refused, cacheless));
}
/// <summary>
@ -3722,10 +3736,14 @@ namespace GameCult.Caching
// The one refusal of a write onto records stored under ids no registered type declares, carried foreign or held read-only.
// It names each record's key, stored id and schema name, never its payload. When the reload that refusal owes failed, the
// refusal carries that failure as its inner exception.
protected CultSchemaConflictException Overwrites(Exception? inner, (string Key, string SchemaId, string SchemaName)[] refused) => new(
// A cacheless writer (a CultMesh single-file write) declares only the catalog entry it writes, and has nothing to reload.
protected CultSchemaConflictException Overwrites(Exception? inner, (string Key, string SchemaId, string SchemaName)[] refused, bool cacheless = false) => new(
$"{(refused.Length == 1 ? "Record" : "Records")} {string.Join(", ", refused.Select(record => $"'{record.Key}' (schema id '{record.SchemaId}', '{record.SchemaName}')"))} in {this}: " +
"no registered type owns this schema id or declares it compatible, so this cache never writes, relabels or removes it. " +
"Declare the id compatible on the type that should own it." +
(cacheless
? "the catalog entry this write publishes neither owns this schema id nor lists it compatible, so the write never relabels or removes it; nothing was written and nothing was reloaded. " +
"List the id as compatible on the schema that should own it."
: "no registered type owns this schema id or declares it compatible, so this cache never writes, relabels or removes it. " +
"Declare the id compatible on the type that should own it.") +
(inner == null ? string.Empty : " Reloading what the write refused failed, so the cache is unchanged."),
refused[0].SchemaId,
refused.Select(record => record.SchemaName).Distinct(StringComparer.Ordinal).ToArray(),
@ -3911,7 +3929,8 @@ namespace GameCult.Caching
// A flush applies this store's staged writes and removals onto what the file holds now, under the lock: two writers never
// interleave bytes, every record this cache did not stage stays as the file has it, and a plain flush is last-writer-wins
// only per staged key. In a store holding a variant it lands only onto the store as this cache last read it (ApplyWriteSet).
// only per staged key. A key whose durable record is under an id this cache does not declare refuses the write, and in a store
// holding a variant it lands only onto the store as this cache last read it (both ApplyWriteSet).
// A file it cannot read (truncated, trailing bytes, not a store, a format it does not read) is refused and left as it is.
public override void PushAll()
{
@ -3921,7 +3940,6 @@ namespace GameCult.Caching
using (AcquireLock(wait: true))
{
var disk = ReadSnapshot();
RefuseForeign(_staged, disk);
var (upserts, removals) = StagedWrites(Array.Empty<CultStoredDocument>(), Array.Empty<CultStoredDocument>());
Wrote(ApplyWriteSet(disk, upserts, removals));
}
@ -3951,7 +3969,6 @@ namespace GameCult.Caching
Entries.Values))
return CultCommitOutcome.Mismatch;
RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), disk);
var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes);
var written = ApplyWriteSet(disk, upserts, removals);
foreach (var entry in request.Deletes)
@ -4013,7 +4030,8 @@ namespace GameCult.Caching
_ = ApplyWriteSet(ReadSnapshot(), upserts, removals);
});
// Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. A store that holds
// Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. First, a key the write
// names whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk). Then a store that holds
// a variant, or would after this write, is written only while its record versions are what this store last read: a variant
// resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions
// only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is
@ -4022,6 +4040,7 @@ namespace GameCult.Caching
// the bytes written.
private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection<CultStagedRecord> upserts, IReadOnlyCollection<string> removals)
{
RefuseUndeclaredOnDisk(durable, upserts, removals);
RefuseWriteIntoMovedVariantStore(durable, upserts, removals);
var durableRecords = durable?.Snapshot.Records ?? Array.Empty<CultPersistedRecord>();
// A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that.
@ -4091,15 +4110,12 @@ namespace GameCult.Caching
if (changed.Count == 0)
return;
var byKey = records.ToDictionary(record => record.Key, StringComparer.Ordinal);
var recordKeys = _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals)
.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).ToArray();
var changedKeys = changed.OrderBy(key => key, StringComparer.Ordinal).ToArray();
Refuse(
recordKeys.Concat(changedKeys),
key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.Snapshot.SchemaCatalog) != null
? null
: ToStoredDocument(record, versions[key], durable!.Snapshot.SchemaCatalog, DeserializePayload, out _),
key => Cache == null ? null : DurableAt(durable, key),
key =>
{
_staged.Remove(key);
@ -4123,34 +4139,52 @@ namespace GameCult.Caching
inner));
}
private void RefuseForeign(IEnumerable<string> written, Durable? disk)
// A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id
// and the ids it lists compatible. A refused key's durable version is what this store has now read of it.
private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection<CultStagedRecord> upserts, IReadOnlyCollection<string> removals)
{
var foreign = ForeignOnDisk(disk);
RefuseForeign(
written,
key => foreign.TryGetValue(key, out var carried) ? carried : null,
Func<string, bool> declares = Cache != null
? Registry.Declares
: id => upserts.Any(staged =>
string.Equals(staged.Entry.SchemaId, id, StringComparison.Ordinal) ||
staged.Entry.CompatibleSchemaIds.Contains(id, StringComparer.Ordinal));
var undeclared = UndeclaredOnDisk(durable, declares);
if (undeclared.Count == 0)
return;
RefuseUndeclaredOnDisk(
_staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals),
key => undeclared.TryGetValue(key, out var carried) ? carried : null,
key => Cache == null ? null : DurableAt(durable, key),
key =>
{
_staged.Remove(key);
IsDirty = _staged.Count > 0;
if (foreign.ContainsKey(key))
_lastRead[key] = disk!.Version(key);
else
_lastRead.Remove(key);
});
_lastRead[key] = durable!.Version(key);
},
cacheless: Cache == null);
}
// The records the file holds that no registered type claims, by key.
private Dictionary<string, CultForeignRecord> ForeignOnDisk(Durable? disk)
// Every durable record under an id the writer does not declare, by key, named by the catalog entry that publishes its id.
private static Dictionary<string, (string Key, string SchemaId, string SchemaName)> UndeclaredOnDisk(Durable? durable, Func<string, bool> declares)
{
var foreign = new Dictionary<string, CultForeignRecord>(StringComparer.Ordinal);
foreach (var record in disk?.Snapshot.Records ?? Array.Empty<CultPersistedRecord>())
var undeclared = new Dictionary<string, (string Key, string SchemaId, string SchemaName)>(StringComparer.Ordinal);
foreach (var record in durable?.Snapshot.Records ?? Array.Empty<CultPersistedRecord>())
{
if (Foreign(record, disk!.Snapshot.SchemaCatalog) is { } carried)
foreign[record.Key] = carried;
if (Undeclared(record, durable!.Snapshot.SchemaCatalog, declares) is { } carried)
undeclared[record.Key] = carried;
}
return foreign;
return undeclared;
}
// The durable record at a key decoded as a load decodes it, at its durable version: null when the store holds none there or
// holds it foreign. The one decode both refusals reload through.
private CultStoredDocument? DurableAt(Durable? durable, string key)
{
var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal));
if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog) != null)
return null;
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _);
}
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:

8
tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs
View file

@ -17,6 +17,8 @@ public sealed class CultMeshSingleFileElementIdTests
{
private const string HexA = "aaaaaaaaaaaa";
private static readonly CultRecordKey Key = new("publication");
// A raw write lands beside the typed record, never over it: neither declares the other's schema id.
private static readonly CultRecordKey RawKey = new("raw");
private string _root = null!;
@ -84,12 +86,12 @@ public sealed class CultMeshSingleFileElementIdTests
var schema = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1");
var payload = new byte[] { 0x90 };
CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload);
CultMesh.WriteSingleFileDocumentPayload(path, RawKey, schema, null, payload);
HeaderOf(path).Should().Be("cultcache.store.v1");
CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor()));
HeaderOf(path).Should().Be("cultcache.store.v3");
CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload);
CultMesh.WriteSingleFileDocumentPayload(path, RawKey, schema, null, payload);
HeaderOf(path).Should().Be("cultcache.store.v3", "the payload's ids are not the writer's to see");
}
@ -102,7 +104,7 @@ public sealed class CultMeshSingleFileElementIdTests
var marked = Path_("marked.cc");
CultMesh.WriteSingleFileDocument(marked, Key, Publication(Descriptor()));
CultMesh.WriteSingleFileDocumentPayload(marked, Key, schema, null, new byte[] { 0x90 });
CultMesh.WriteSingleFileDocumentPayload(marked, RawKey, schema, null, new byte[] { 0x90 });
HeaderOf(marked).Should().Be("cultcache.store.v3", "a whole marked file is still written and stays marked");
}

11
tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs
View file

@ -183,15 +183,20 @@ public sealed class CultMeshSingleFileStoreTests
Snapshot(path).Records.Should().Contain(record => record.Variant != null);
var key = new CultRecordKey(target switch { "variant" => "v", "base" => "a", _ => "z" });
var conflict = Assert.Throws<CultWriteConflictException>(() =>
void Write()
{
if (typed)
CultMesh.WriteSingleFileDocument(path, key, PublicationOf("z"));
else
CultMesh.WriteSingleFileDocumentPayload(path, key, Raw, null, new byte[] { 0x90 });
})!;
}
conflict.Message.Should().Contain("CultCache");
// A raw write at the variant's or the base's key names a record under an id its entry does not declare, and that refusal is
// checked first.
if (!typed && target != "unrelated")
Assert.Throws<CultSchemaConflictException>(Write)!.RecordKey.Should().Be(key.Value);
else
Assert.Throws<CultWriteConflictException>(Write)!.Message.Should().Contain("CultCache");
File.ReadAllBytes(path).Should().Equal(bytes);
}