From 3a18f01a831f09154e8b6a6574b002e7cdd85f53 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 04:03:41 +0200 Subject: [PATCH 01/13] Caching: a removal of a key the cache does not hold is a check-only key judged under the store's lock; ExpectUnchanged compares foreign records too Remove and Commit of an unheld key no longer read the store at the call: the key travels as CultCommitRequest.Checks (commit) or a staged check (flush), and each writable store judges it in RefuseUndeclaredOnDisk with the keys it writes, never removing it. A batch that only checks writes no file. CultForeignRecord.StoredVersion records the version a store read, and the one ConditionsHold takes the store's foreign records. Folds spec commits 1 and 2 (their hunks share the same methods). The src projects build; tests/GameCult.Caching.Tests does not at this commit (CultCommitRequest gained a parameter; the next commit updates the test). Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 47 +++-- src/GameCult.Caching/CultCache.cs | 179 ++++++++++++++---- 2 files changed, 178 insertions(+), 48 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 71140019..ad77caa8 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -182,10 +182,17 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore return CultCommitOutcome.Contended; var (manifest, catalogBytes) = ReadManifest(); if (request.HasConditions && !request.ConditionsHold( - manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values)) + manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords)) return CultCommitOutcome.Mismatch; - RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(StagedKeys)); + RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys)); + if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) + { + // Nothing to write: the batch only judged keys it does not hold. + MarkFlushSucceeded(); + return CultCommitOutcome.Committed; + } + var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); var previousDirtyKeys = _dirtyKeys.Keys.ToArray(); var previousDeletedKeys = _deletedKeys.Keys.ToArray(); @@ -230,23 +237,41 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore ThrowIfReadOnly(); Held(() => { - if (!IsDirty) + if (!IsDirty && StagedChecks.Count == 0) return; Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); RefuseUndeclaredOnDisk(manifest, StagedKeys); - WriteGeneration(manifest, catalogBytes); + if (IsDirty) + WriteGeneration(manifest, catalogBytes); + else + MarkFlushSucceeded(); }); } - private IEnumerable StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys); + // Every key the next flush or commit judges: the staged writes and removals, and the staged checks. + private IEnumerable StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys).Concat(StagedChecks); - // The manifest as it is now, read without the commit lease: nothing is written after the read, the manifest is only ever - // replaced whole, and a read-only store must not create the lease. - protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) => - Held(() => RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys)); + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the commit lease + // (StageChecks); a read-only store has neither, so it answers from its view and never creates the lease. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } + + // A manifest record no registered type claims, listed at the version its index record names. + private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection catalog) + { + if (Foreign(record, catalog) is not { } carried) + return null; + carried.StoredVersion = VersionOf(record.Payload); + return carried; + } // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a // record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every @@ -261,7 +286,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, - key => HeldAt(key) is { } held ? Foreign(held, currentManifest.SchemaCatalog) : null, + key => HeldAt(key) is { } held ? ForeignAt(held, currentManifest.SchemaCatalog) : null, key => { _dirtyKeys.TryRemove(key, out _); @@ -365,7 +390,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var claimed = new List(records.Length); foreach (var metadata in records) { - if (Foreign(metadata, catalogEntries) is { } carried) + if (ForeignAt(metadata, catalogEntries) is { } carried) foreign.Add(carried); else claimed.Add(metadata); diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2470381c..ede033dc 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -179,6 +179,10 @@ namespace GameCult.Caching public string SchemaName { get; } public string StoredAt { get; } + + // The version of the record's stored bytes when its store read it, in that store's spelling; null when the store kind + // did not give one. ExpectUnchanged compares it with the version on disk at the commit. + internal string? StoredVersion { get; set; } } public sealed class CultPersistedStoreSnapshot @@ -1860,24 +1864,40 @@ namespace GameCult.Caching internal CultCommitRequest( IReadOnlyList upserts, IReadOnlyList deletes, + IReadOnlyList checks, IReadOnlyList<(CultRecordKey Key, string? Version)> expected, bool expectUnchanged) { Upserts = upserts; Deletes = deletes; + Checks = checks; Expected = expected; ExpectUnchanged = expectUnchanged; } public IReadOnlyList Upserts { get; } public IReadOnlyList Deletes { get; } + + /// + /// Keys the batch removes that its cache does not hold. Such a removal removes nothing, so it is never in + /// ; but a store that holds a record at one of these keys under an id no registered type declares + /// refuses the commit, as it refuses a write of that record. A judges these keys + /// under its lock, with the keys it writes, and never removes them: a record a declared type reads stays where it is. + /// + public IReadOnlyList Checks { get; } public IReadOnlyList<(CultRecordKey Key, string? Version)> Expected { get; } public bool ExpectUnchanged { get; } public bool HasConditions => Expected.Count > 0 || ExpectUnchanged; - // durable is the version of each record the store holds on disk now, by key; observed is what its cache last loaded or - // committed. Identity is the record's version, the SHA-256 of its stored bytes, which names its schema id and storedAt too. - public bool ConditionsHold(IReadOnlyDictionary durable, IEnumerable observed) + /// + /// Whether the batch's conditions hold. is the version of each record the store holds on disk now, + /// by key; is what the store's cache last loaded or committed, and the + /// foreign records the store carries as it last read them (; empty for a home + /// with none). Under each foreign record compares against the version it had when the store + /// read it, as a record the cache holds compares against its own. Identity is the record's version, the SHA-256 of its stored + /// bytes, which names its schema id and storedAt too. + /// + public bool ConditionsHold(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign) { foreach (var (key, version) in Expected) { @@ -1893,6 +1913,7 @@ namespace GameCult.Caching .OrderBy(pair => pair.Key, StringComparer.Ordinal) .SequenceEqual(observed .Select(stored => (Key: stored.Key.Value, Version: stored.StoredVersion)) + .Concat(foreign.Select(record => (record.Key, Version: record.StoredVersion))) .OrderBy(pair => pair.Key, StringComparer.Ordinal)); } } @@ -2291,7 +2312,7 @@ namespace GameCult.Caching { foreach (var (store, _) in _stores) { - if (!store.IsReadOnly && store.IsDirty) + if (!store.IsReadOnly && (store.IsDirty || store.HasStagedChecks)) store.PushAll(); } return true; @@ -2352,8 +2373,15 @@ namespace GameCult.Caching return key; })); - // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares, - // read-only or foreign. False when the cache holds nothing there and no store carries a foreign record there. + /// + /// Removes the record at ; false when the cache holds nothing there. + /// + /// + /// A record stored under an id no registered type declares, read-only or foreign, is never removed: removing one the cache + /// holds is refused with now. A key the cache does not hold costs nothing here and + /// returns false; each writable store judges it at its next flush or commit, under its lock, and refuses that flush or + /// commit if it holds a record there under such an id (a read-only store answers now, from what it last loaded). + /// public bool Remove(CultRecordKey key) => Held(() => { if (!_entries.TryGetValue(key.Value, out var existing)) @@ -2467,7 +2495,7 @@ namespace GameCult.Caching { Held(() => { - if (FlushAttachedStoresOnDispose && IsDirty) + if (FlushAttachedStoresOnDispose && (IsDirty || _stores.Any(entry => entry.Store.HasStagedChecks))) FlushAllBackingStores(); return true; }); @@ -2631,18 +2659,31 @@ namespace GameCult.Caching : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) : StampFlatten(op.Key)) .ToArray(); - RefuseRemovalOfUndeclared(batch.Operations.Values + // A removal of a key the cache does not hold removes nothing: it is a check the commit's store judges under its lock. + var checks = batch.Operations.Values .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) .Select(op => op.Key.Value) - .ToArray()); + .ToArray(); var deletes = batch.Operations.Values .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) .Select(op => _entries[op.Key.Value]) .ToArray(); return Admit(upserts, deletes, null, (home, resolved) => { + // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. + if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) + home = only[0].Store; + // Every other store answers the checks now, from what it last loaded: the batch commits through one store. + foreach (var (store, _) in _stores) + { + if (store == home) + continue; + foreach (var key in checks) + store.RefuseUndeclared(key); + } + // What the store holds is what the hold resolved: a variant's delta beside its resolved view. - var request = new CultCommitRequest(resolved, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); + var request = new CultCommitRequest(resolved, deletes, checks, batch.Expected.ToArray(), batch.ExpectsUnchanged); if (home != null) { if (request.HasConditions && home.IsDirty) @@ -2653,14 +2694,14 @@ namespace GameCult.Caching if (request.HasConditions && _stores.Count > 0) throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes."); var inMemory = _entries.Values.ToDictionary(entry => entry.Key.Value, entry => entry.StoredVersion, StringComparer.Ordinal); - return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + return request.ConditionsHold(inMemory, _entries.Values, Array.Empty()) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; }); }); } // A removal of a key the cache does not hold removes nothing, so no write carries it to a store; but a store that holds a // record there under an id no registered type declares refuses it, as it refuses every write of that record, rather than let - // the caller believe it was removed. + // the caller believe it was removed. Each store judges it its own way (CacheBackingStore.RefuseRemovalOfUndeclared). private void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { if (keys.Count == 0) @@ -3529,13 +3570,21 @@ namespace GameCult.Caching public abstract void Delete(CultStoredDocument entry); public abstract void PushAll(); - // One durable step under the store's exclusive lock; conditions are evaluated against what is on disk then. - // A failure restores the store's staged view. + /// + /// One durable step under the store's exclusive lock: the batch and every staged write land onto the store as it is then, + /// and the batch's conditions are evaluated against it. A failure restores the store's staged view. + /// + /// + /// An implementation judges , and its own , under that + /// lock with the keys it writes: a key whose record is under an id no registered type declares refuses the commit, and a + /// checked key is never removed. Pass to + /// . + /// public abstract CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait); public virtual void Dispose() { - if (FlushOnDispose && IsDirty && !IsReadOnly) + if (FlushOnDispose && (IsDirty || HasStagedChecks) && !IsReadOnly) { PushAll(); } @@ -3654,16 +3703,52 @@ namespace GameCult.Caching throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } - // The apply's refusal for removals of keys the cache does not hold. Such a removal writes nothing, so no write carries it to - // where the store is read; this reads it instead. A key whose durable record is under an id no registered type declares is - // refused and reloaded as the apply refuses and reloads it, including a record another writer put there after the load. - // A store kind that cannot read its durable state answers from its view (RefuseUndeclared). + /// + /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal + /// removes nothing, but a store that holds a record at one of these keys under an id no registered type declares must + /// refuse it rather than let the caller believe the record is gone. + /// + /// + /// + /// The default refuses now, from this store's view: the foreign records its last load listed and the records it holds + /// (). It reads nothing, so it cannot see a record another writer put there after that load. + /// + /// + /// A writable store kind that reads its durable state under a lock overrides this to call , and + /// judges at its next or as it judges + /// : under the lock, with the keys it writes, refused through the same refusal, never + /// removed. It calls when that step succeeds, which clears them. A read-only store has no + /// such step and keeps the default. Both shipped store kinds do this. + /// + /// protected internal virtual void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { foreach (var key in keys) RefuseUndeclared(key); } + // Keys a removal named that this store's cache does not hold, waiting for this store's next flush or commit to judge them. + // A refusal takes back the keys it refused; a successful flush or commit clears them. + private readonly HashSet _checks = new(StringComparer.Ordinal); + + /// + /// The keys staged for this store's next flush or commit to judge. They are not writes: + /// does not count them. + /// + protected IReadOnlyCollection StagedChecks => _checks; + + /// + /// Stages keys for this store's next flush or commit to judge without removing them. See + /// . + /// + protected void StageChecks(IEnumerable keys) => Held(() => + { + foreach (var key in keys) + _checks.Add(key); + }); + + internal bool HasStagedChecks => _checks.Count > 0; + // A stored record under an id the writer does not declare, named by the catalog entry that publishes the id. Header only: no // payload is decoded. Null for a record under a declared id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( @@ -3739,7 +3824,10 @@ namespace GameCult.Caching } foreach (var key in refused) + { unstage(key); + _checks.Remove(key); + } foreach (var held in dropped) Entries.TryRemove(held.Key.Value, out _); foreach (var stored in loaded) @@ -3790,9 +3878,11 @@ namespace GameCult.Caching refused[0].Key, inner); + // A flush or commit succeeded: nothing staged is left, writes or checks. protected void MarkFlushSucceeded() { IsDirty = false; + _checks.Clear(); } } @@ -3917,6 +4007,7 @@ namespace GameCult.Caching var version = versions[record.Key] = durable.Version(record.Key); if (Foreign(record, snapshot.SchemaCatalog) is { } carried) { + carried.StoredVersion = version; foreign.Add(carried); continue; } @@ -3981,7 +4072,7 @@ namespace GameCult.Caching { var disk = ReadSnapshot(); var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); - Wrote(ApplyWriteSet(disk, upserts, removals)); + Wrote(ApplyWriteSet(disk, upserts, removals, Array.Empty())); } _staged.Clear(); @@ -4006,11 +4097,12 @@ namespace GameCult.Caching var disk = ReadSnapshot(); if (request.HasConditions && !request.ConditionsHold( (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal), - Entries.Values)) + Entries.Values, + ForeignRecords)) return CultCommitOutcome.Mismatch; var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); - var written = ApplyWriteSet(disk, upserts, removals); + var written = ApplyWriteSet(disk, upserts, removals, request.Checks); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) @@ -4067,20 +4159,24 @@ namespace GameCult.Caching internal void ApplyWriteSet(IReadOnlyCollection upserts, IReadOnlyCollection removals) => Held(() => { using (AcquireLock(wait: true)) - _ = ApplyWriteSet(ReadSnapshot(), upserts, removals); + _ = ApplyWriteSet(ReadSnapshot(), upserts, removals, Array.Empty()); }); // Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. First, a key the write - // names whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk). Then a store that holds + // names or checks whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk); a + // write that then has nothing to write (it only checked keys) writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of // the bytes written. - private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet( + Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) { - RefuseUndeclaredOnDisk(durable, upserts, removals); + RefuseUndeclaredOnDisk(durable, upserts, removals, checks); + if (upserts.Count == 0 && removals.Count == 0) + return Array.Empty<(CultStagedRecord, string)>(); RefuseWriteIntoMovedVariantStore(durable, upserts, removals); var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. @@ -4180,17 +4276,22 @@ namespace GameCult.Caching inner)); } - // The file as it is now, read as a pull reads it, without the lock: nothing is written after the read, and the file is only - // ever replaced whole. - protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) => - Held(() => RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares)); + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the lock (StageChecks); + // a read-only store has neither, so it answers from its view and never creates the lock file. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } // A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id - // and the ids it lists compatible. - private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) => + // and the ids it lists compatible. The keys judged are every key staged, written, removed or checked. + private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, - _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals), + _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), Cache != null ? Registry.Declares : id => upserts.Any(staged => @@ -4231,10 +4332,14 @@ namespace GameCult.Caching } // The durable record at a key as a load lists it when no registered type claims its id; null otherwise. - private CultForeignRecord? ForeignAt(Durable? durable, string key) => - durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is { } record - ? Foreign(record, durable.Snapshot.SchemaCatalog) - : null; + private CultForeignRecord? ForeignAt(Durable? durable, string key) + { + if (durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is not { } record || + Foreign(record, durable.Snapshot.SchemaCatalog) is not { } carried) + return null; + carried.StoredVersion = durable.Version(key); + return carried; + } // The durable record at a key decoded as a load decodes it, at its durable version: null when the store holds none there or // holds it foreign. The one decode both refusals reload through. From 2067dcd1a24832301ad34f702a7430f7b20ac100 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 04:03:41 +0200 Subject: [PATCH 02/13] Tests: unheld-removal checks judged at the commit and flush, checks never remove, ExpectUnchanged over foreign records, delisting and variant-refusal listing pinned Co-Authored-By: Claude Sonnet 5.5 --- .../ForeignRecordTests.cs | 174 +++++++++++++++++- 1 file changed, 168 insertions(+), 6 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 1f417f2c..ddb708ad 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1011,10 +1011,10 @@ namespace GameCult.Caching.Tests store.PushAll(); break; case DirectWrite.CommitUpsert: - store.CommitBatch(new CultCommitRequest(new[] { mine }, Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + store.CommitBatch(new CultCommitRequest(new[] { mine }, Array.Empty(), Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); break; case DirectWrite.CommitDelete: - store.CommitBatch(new CultCommitRequest(Array.Empty(), new[] { held }, Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + store.CommitBatch(new CultCommitRequest(Array.Empty(), new[] { held }, Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); break; } })!; @@ -1026,9 +1026,10 @@ namespace GameCult.Caching.Tests Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("theirs")); } - // A removal of a foreign record, which the cache does not hold, is refused typed: the caller is never told it removed what the - // store still carries. The store is read as it is at the removal, so a foreign record another writer added after the load - // is refused too, and listed foreign from then on. + // A removal of a foreign record, which the cache does not hold, is refused typed where the store judges it under its lock: at + // the commit that carries it, or the flush that follows a Remove, which itself costs nothing and returns false. The caller is + // never told a write removed what the store still carries. The store is read at that judgement, so a foreign record another + // writer added after the load is refused too, and listed foreign from then on. [Test] public void RemovingAForeignRecordIsRefusedNotReportedAsDone([Values] bool directory, [Values] bool batch, [Values] bool appearedAfterLoad) { @@ -1054,7 +1055,10 @@ namespace GameCult.Caching.Tests if (batch) cache.Commit(stage => stage.Remove(W)); else - cache.Remove(W); + { + Assert.That(cache.Remove(W), Is.False, "a key the cache does not hold costs nothing at the call"); + cache.FlushAsync().GetAwaiter().GetResult(); + } })!; Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); Assert.That(cache.IsDirty, Is.False); @@ -1063,5 +1067,163 @@ namespace GameCult.Caching.Tests Assert.That(Fingerprint(path), Is.EqualTo(before)); } } + + // A write that removes a key the cache does not hold is judged under the store's lock, after the load: a foreign record another + // writer committed in between refuses the whole commit, typed, and nothing the commit would have written lands. + [Test] + public void AForeignRecordThatLandsBetweenTheLoadAndTheCommitRefusesTheCommitThatRemovesIt([Values] bool directory) + { + var path = PathOf("window.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, DeckOnly, directory); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => a.Commit(batch => + { + batch.Upsert(Deck, DeckOf(Canary), D); + batch.Remove(W); + }))!; + + Assert.Multiple(() => + { + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(Fingerprint(path), Is.EqualTo(before), "nothing was written, d included"); + Assert.That(a.IsDirty, Is.False); + }); + } + + // Removing a key nothing holds reads no store: the judgement is the flush's, so a store that cannot be read costs the removal + // nothing and the flush is where it is read. + [Test] + public void RemovingAnUnheldKeyReadsNothingUntilTheFlush([Values] bool directory) + { + var path = Seeded("unreadable.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var original = File.ReadAllBytes(path); + File.WriteAllBytes(path, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + try + { + Assert.DoesNotThrow(() => Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False)); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + finally + { + File.WriteAllBytes(path, original); + } + } + + // A checked key is judged and never removed: another writer's declared record at it stays where it is. + [Test] + public void ACheckedKeyIsNeverRemoved([Values] bool directory) + { + var path = PathOf("checked.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, Full, directory); + Assert.That(a.Remove(W), Is.False); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); + + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged"); + } + + // ExpectUnchanged compares the foreign records a store carries at the versions it read them: it holds right after the load, + // and fails, writing nothing, once another writer changed one. + [Test] + public void ExpectUnchangedHoldsOverAStoreCarryingAForeignRecord([Values] bool directory) + { + var path = Seeded("expect-unchanged.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("e"), E); + }), Is.True, "nothing changed since the load"); + + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("changed"), W)); + var before = Fingerprint(path); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("f"), new CultRecordKey("f")); + }), Is.False, "another writer changed the foreign record"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A foreign key listed at the load that another writer then replaced with a record this cache reads but does not declare: the + // refused removal (judged at the flush) reloads it read-only and takes it off the foreign list. + [Test] + public void AListedForeignKeyBecomesReadOnlyWhenARemovalOfItIsRefused([Values] bool directory) + { + var path = Seeded("delist.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + Move(path, W, "moved", directory); + + Assert.That(a.Remove(W), Is.False); + var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; + RefusedOnDisk(refusal, W); + + Assert.That(StoreOf(a).ForeignRecords, Is.Empty, "no longer foreign"); + Assert.That(a.Get(W), Is.Not.Null, "held, read-only"); + Assert.Throws(() => a.Remove(W), "refused at the call now: the cache holds it"); + } + + // A single-file store holding a variant refuses a write after another writer changed it; a changed key whose record this cache + // does not read is listed as foreign by that refusal, as a load lists it. + [Test] + public void AVariantStoreRefusalListsTheRecordItRefusedAsForeign([Values(Write.Flush, Write.Commit)] Write write) + { + var path = PathOf("variant-refusal.cc"); + var gears = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear) }); + var withWidget = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear), Widget }); + using var a = Open(path, gears, directory: false); + var baseKey = new CultRecordKey("laser"); + a.Commit(batch => + { + batch.Upsert(typeof(VariantTests.VariantGear), new VariantTests.VariantGear { Name = "laser", Power = 10, Code = "l1" }, baseKey); + batch.UpsertVariant(new CultRecordKey("laser-big"), baseKey, new[] + { + a.Override(nameof(VariantTests.VariantGear.Name), "laser big"), + a.Override(nameof(VariantTests.VariantGear.Power), 25), + a.Override(nameof(VariantTests.VariantGear.Code), "laser big") + }); + }); + using (var b = Open(path, withWidget, directory: false)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + Assert.That(StoreOf(a).ForeignRecords, Is.Empty); + + var conflict = Assert.Throws(() => + { + var next = new VariantTests.VariantGear { Name = "next", Power = 1, Code = "n" }; + var key = new CultRecordKey("n"); + if (write == Write.Flush) + { + a.UpsertAsync(typeof(VariantTests.VariantGear), next, key).GetAwaiter().GetResult(); + a.FlushAsync().GetAwaiter().GetResult(); + } + else + { + a.Commit(batch => batch.Upsert(typeof(VariantTests.VariantGear), next, key)); + } + })!; + + Assert.That(conflict.ChangedKeys, Does.Contain(W.Value)); + var listed = StoreOf(a).ForeignRecords.Single(); + Assert.That((listed.Key, listed.SchemaId), Is.EqualTo((W.Value, WidgetId)), "the refused record is listed as the store holds it"); + Assert.That(a.Get(W), Is.Null); + } } } From 2e3a5b8e16357bf854d40f66c33f0f7a6b984ac0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 04:03:42 +0200 Subject: [PATCH 03/13] Contracts: an unheld removal is a check-only key judged under the store's lock; ExpectUnchanged compares foreign records Co-Authored-By: Claude Sonnet 5.5 --- .../Contracts/cultcache-persistence-format.md | 2 +- .../Contracts/cultcache-store-composition.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index ec9e5b03..69de1c75 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,7 +206,7 @@ A record that resolves to no local type is a foreign record, for example one of have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords` in C#), and the cache never holds it. Writes of other records proceed and copy it forward as the file holds it. A write that would replace or remove it is refused with the typed schema conflict -naming its key and id, and nothing is written. A removal of a key the cache does not hold is judged against the store as it is at the removal, so it is refused also when the foreign record arrived after the load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. +naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 1d9d709e..740b5f73 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -263,6 +263,11 @@ refuses to replace a file whose header it cannot read, and leaves it as it was. before anything is written, and that record is reloaded as the store holds it: dropped when foreign, held read-only when a type reads it (cultcache-persistence-format.md, ownership by declaration). +- A removal of a key the cache does not hold travels as a check-only key: a + commit carries it in the request, a flush stages it, and either store kind + judges it under its lock with the keys it writes, never removing it. A batch + that only checks keys writes no file. `ExpectUnchanged` compares the foreign + records a store carries at the versions it read them. - A single-file store holding a variant (the directory store refuses variants) is written only from what its writer last read. A record that moved refuses the whole write with the typed write conflict, and the refused and moved keys From 649ac1860e940f9dadd31f6f0df154dc255836b9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 04:09:43 +0200 Subject: [PATCH 04/13] Tests: a flush clears the checks it judged and a refusal forgets the check it refused Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/ForeignRecordTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index ddb708ad..63835ee4 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1063,6 +1063,7 @@ namespace GameCult.Caching.Tests Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); Assert.That(cache.IsDirty, Is.False); Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult(), "a refusal forgets the check it refused"); Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False, "a key nothing holds is still no removal"); Assert.That(Fingerprint(path), Is.EqualTo(before)); } @@ -1123,7 +1124,7 @@ namespace GameCult.Caching.Tests var path = PathOf("checked.cc"); using (var seed = Open(path, Full, directory)) seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); - using var a = Open(path, Full, directory); + using var a = Open(path, DeckOnly, directory); Assert.That(a.Remove(W), Is.False); using (var b = Open(path, Full, directory)) b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); @@ -1132,7 +1133,9 @@ namespace GameCult.Caching.Tests Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); - Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged"); + using (var c = Open(path, Full, directory)) + c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged: the next one has nothing to judge"); } // ExpectUnchanged compares the foreign records a store carries at the versions it read them: it holds right after the load, From cec09806b007af656b0f95bf07c127ad56946ebc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 05:06:19 +0200 Subject: [PATCH 05/13] Caching: an empty direct flush still applies as before; only a write that checked keys and has nothing else to write writes no file. Tests: the delisting writer declares the widget; the checked-key flush also writes Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 4 ++-- tests/GameCult.Caching.Tests/ForeignRecordTests.cs | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index ede033dc..dd88c346 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -4164,7 +4164,7 @@ namespace GameCult.Caching // Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. First, a key the write // names or checks whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk); a - // write that then has nothing to write (it only checked keys) writes no file. Then a store that holds + // write that then has nothing to write but keys it checked writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is @@ -4175,7 +4175,7 @@ namespace GameCult.Caching Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) { RefuseUndeclaredOnDisk(durable, upserts, removals, checks); - if (upserts.Count == 0 && removals.Count == 0) + if (upserts.Count == 0 && removals.Count == 0 && (checks.Any() || StagedChecks.Count > 0)) return Array.Empty<(CultStagedRecord, string)>(); RefuseWriteIntoMovedVariantStore(durable, upserts, removals); var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 63835ee4..10742ad0 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1125,6 +1125,7 @@ namespace GameCult.Caching.Tests using (var seed = Open(path, Full, directory)) seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); using var a = Open(path, DeckOnly, directory); + a.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); Assert.That(a.Remove(W), Is.False); using (var b = Open(path, Full, directory)) b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); @@ -1132,6 +1133,7 @@ namespace GameCult.Caching.Tests Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" }), "the staged write landed beside the checked key"); Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); using (var c = Open(path, Full, directory)) c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); @@ -1173,7 +1175,9 @@ namespace GameCult.Caching.Tests var path = Seeded("delist.cc", directory); using var a = Open(path, DeckOnly, directory); Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); - Move(path, W, "moved", directory); + // The writer declares the widget's id, so it may replace the widget with a deck record under an id a reads but does not declare. + using (var other = Open(path, CultDocumentRegistry.ForTypes(new[] { MovingDeck, Widget }), directory)) + Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", "moved")), W)), Is.True); Assert.That(a.Remove(W), Is.False); var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; From 7b604d41bd58757c5305e3eac1e0236c446b7873 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 07:36:45 +0200 Subject: [PATCH 06/13] Tests: check-only writes write no file and clear their checks, dispose and read-only stores judge them, a non-home store answers from its view and the home under its lock Co-Authored-By: Claude Sonnet 5.5 --- .../ForeignRecordTests.cs | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 10742ad0..44b260bc 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1232,5 +1232,147 @@ namespace GameCult.Caching.Tests Assert.That((listed.Key, listed.SchemaId), Is.EqualTo((W.Value, WidgetId)), "the refused record is listed as the store holds it"); Assert.That(a.Get(W), Is.Null); } + + private static CacheBackingStore StoreAt(string path, bool directory, bool readOnly) => directory + ? new DirectoryMessagePackBackingStore(path, readOnly: readOnly) + : new SingleFileMessagePackBackingStore(path, readOnly); + + // A batch with nothing to write and no key to check commits through no store: a staged write waits for its own flush. + [Test] + public void ABatchWithNothingToWriteOrCheckCommitsNothingToTheStore([Values] bool directory) + { + var path = Seeded("empty-batch.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); + + Assert.That(cache.Commit(batch => { }), Is.True); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "w" }), "e is still only staged"); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" })); + } + + // A write that only checked keys has nothing to write: no file is replaced and no generation is written, by a commit or a flush. + [Test] + public void AWriteThatOnlyCheckedKeysWritesNoFile([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-only.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var past = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc); + File.SetLastWriteTimeUtc(path, past); + var before = Fingerprint(path); + + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True); + else + { + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(File.GetLastWriteTimeUtc(path), Is.EqualTo(past), "the store was not replaced"); + Assert.That(cache.IsDirty, Is.False); + } + + // A flush or commit that only checked keys clears every check it judged: the next one has nothing to judge, so it does not + // refuse a record another writer put at a checked key afterwards. + [Test] + public void AWriteThatOnlyCheckedKeysClearsTheChecksItJudged([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-cleared.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var staged = new CultRecordKey("staged"); + Assert.That(cache.Remove(staged), Is.False); + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("requested"))), Is.True); + else + cache.FlushAsync().GetAwaiter().GetResult(); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), staged)); + + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + + // A disposed cache and a disposed store judge the checks a removal staged, as their flush does. + [Test] + public void DisposingAStoreJudgesTheChecksARemovalStaged([Values] bool directory, [Values] bool viaCache) + { + var path = Seeded("dispose.cc", directory); + var cache = Open(path, DeckOnly, directory); + cache.FlushAttachedStoresOnDispose = true; + StoreOf(cache).FlushOnDispose = true; + Assert.That(cache.Remove(W), Is.False); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => + { + if (viaCache) + cache.Dispose(); + else + StoreOf(cache).Dispose(); + })!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } + + // A read-only store has no lock to judge under, so it answers a removal of a key the cache does not hold at the call, from + // what it last loaded. + [Test] + public void AReadOnlyStoreRefusesTheRemovalOfAForeignRecordAtTheCall([Values] bool directory) + { + var path = Seeded("read-only.cc", directory); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = DeckOnly, UseDirectoryStore = directory, ReadOnly = true }); + + var refusal = Assert.Throws(() => cache.Remove(W))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + } + + // A batch that only checks keys commits through the one writable store; every other store answers the checks from its view. + [Test] + public void AStoreThatIsNotTheHomeAnswersTheChecksFromItsView([Values] bool directory) + { + var writable = PathOf("home.cc"); + using (var seed = Open(writable, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var readOnly = PathOf("other.cc"); + using (var seed = Open(readOnly, Full, directory)) + seed.Commit(batch => + { + batch.Upsert(Deck, DeckOf("e"), E); + batch.Upsert(Widget, WidgetOf("w"), W); + }); + using var cache = new CultCache(DeckOnly); + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false)); + cache.AddBackingStore(StoreAt(readOnly, directory, readOnly: true)); + var before = Fingerprint(writable); + + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Remove(W)))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(writable), Is.EqualTo(before)); + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True, "a key no store holds is free"); + } + + // The home store judges a check under its lock, from the record it holds then: a foreign record it listed at the load that + // another writer since replaced with a declared one no longer refuses the commit, and is not removed. + [Test] + public void TheHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var path = Seeded("home-judges.cc", directory); + using var cache = Open(path, DeckOnly, directory); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.That(cache.Commit(batch => batch.Remove(W)), Is.True); + + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } } } From d0775706e5e5caf1c803c11438fb6800d047f30d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 07:39:32 +0200 Subject: [PATCH 07/13] Tests: the non-home store case gives its second store a home type of its own Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Caching.Tests/ForeignRecordTests.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 44b260bc..978421df 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -18,6 +18,8 @@ namespace GameCult.Caching.Tests private static readonly Type ClaimingDeck = Emit("ForeignSuiteClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v1", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); private static readonly Type LateClaimingDeck = Emit("ForeignSuiteLateClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); private static readonly Type Widget = Emit("ForeignSuiteWidget", "tests.foreign_widget", "tests.foreign_widget.v1", new[] { new Field("Label", typeof(string), 0, IsName: true) }); + // A second type a cache registers, so that a second backing store has a home of its own. + private static readonly Type Gizmo = Emit("ForeignSuiteGizmo", "tests.foreign_gizmo", "tests.foreign_gizmo.v1", new[] { new Field("Tag", typeof(string), 0, IsName: true) }); // A build that has both types, and one that has only the deck: to it the widget is foreign. private static readonly CultDocumentRegistry Full = CultDocumentRegistry.ForTypes(new[] { Deck, Widget }); @@ -1340,15 +1342,15 @@ namespace GameCult.Caching.Tests using (var seed = Open(writable, Full, directory)) seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); var readOnly = PathOf("other.cc"); - using (var seed = Open(readOnly, Full, directory)) + using (var seed = Open(readOnly, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) seed.Commit(batch => { - batch.Upsert(Deck, DeckOf("e"), E); + batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g")); batch.Upsert(Widget, WidgetOf("w"), W); }); - using var cache = new CultCache(DeckOnly); - cache.AddBackingStore(StoreAt(writable, directory, readOnly: false)); - cache.AddBackingStore(StoreAt(readOnly, directory, readOnly: true)); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(readOnly, directory, readOnly: true), Gizmo); var before = Fingerprint(writable); var refusal = Assert.Throws(() => cache.Commit(batch => batch.Remove(W)))!; From 5a3cf7e9a9f1e413e2305a9716e065e7dca6d913 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 12 Oct 2026 07:46:10 +0200 Subject: [PATCH 08/13] Tests: a cache disposed with one writable store holding staged checks beside a read-only one still judges them Co-Authored-By: Claude Sonnet 5.5 --- .../ForeignRecordTests.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 978421df..a598ed98 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1376,5 +1376,25 @@ namespace GameCult.Caching.Tests Assert.That(Stored(path, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); } + + // A cache disposed with several stores flushes when any one of them has checks staged, not only when all do (a read-only store + // never has any). + [Test] + public void DisposingACacheFlushesWhenOnlyOneStoreHasChecksStaged([Values] bool directory) + { + var writable = Seeded("dispose-home.cc", directory); + var other = PathOf("dispose-other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo }), directory)) + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })) { FlushAttachedStoresOnDispose = true }; + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: true), Gizmo); + Assert.That(cache.Remove(W), Is.False); + + var refusal = Assert.Throws(() => cache.Dispose())!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } } } From 1893410fe1625ecc43be980be964cd2e76f4123a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:07:19 +0200 Subject: [PATCH 09/13] Every writable store judges a batch's unheld removals under its own lock; a foreign record's version comes through Foreign CacheBackingStore.JudgeChecks (default: the view; single-file and directory stores read under their lock) replaces Land's view-only loop over non-home stores. Foreign takes the version the store read; CultForeignRecord.StoredVersion is get-only and set by the ctor, and every assignment after Foreign is deleted. One commit carries both spec commits 1 and 2 (the edits share hunks); unverified until the final job. Co-Authored-By: Claude Opus 5.5 --- .../DirectoryMessagePackBackingStore.cs | 26 ++++-- src/GameCult.Caching/CultCache.cs | 81 ++++++++++++++----- 2 files changed, 80 insertions(+), 27 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index ad77caa8..a5e2b18d 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -264,15 +264,27 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore StageChecks(keys); } - // A manifest record no registered type claims, listed at the version its index record names. - private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection catalog) + // A read-only store answers a batch's checks from its view and never creates the lease; a writable one reads the manifest + // under the commit lease. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) { - if (Foreign(record, catalog) is not { } carried) - return null; - carried.StoredVersion = VersionOf(record.Payload); - return carried; + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + Directory.CreateDirectory(_manifestFile.DirectoryName!); + using var commitLease = AcquireCommitLease(wait, create: true); + if (commitLease == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys); + return CultCommitOutcome.Committed; + }); } + // A manifest record no registered type claims, listed at the version its index record names. + private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection catalog) => + Foreign(record, catalog, VersionOf(record.Payload)); + // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a // record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. @@ -283,7 +295,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore RefuseUndeclaredOnDisk( written, key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Declares) : null, - key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog) == null + key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, key => HeldAt(key) is { } held ? ForeignAt(held, currentManifest.SchemaCatalog) : null, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index dd88c346..a8e0e739 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -163,12 +163,13 @@ namespace GameCult.Caching /// public sealed class CultForeignRecord { - internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt) + internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt, string storedVersion) { Key = key; SchemaId = schemaId; SchemaName = schemaName; StoredAt = storedAt; + StoredVersion = storedVersion; } public string Key { get; } @@ -180,9 +181,9 @@ namespace GameCult.Caching public string StoredAt { get; } - // The version of the record's stored bytes when its store read it, in that store's spelling; null when the store kind - // did not give one. ExpectUnchanged compares it with the version on disk at the commit. - internal string? StoredVersion { get; set; } + // The version of the record's stored bytes when its store read it, in that store's spelling, given by the store through + // CacheBackingStore.Foreign. ExpectUnchanged compares it with the version on disk at the commit. + internal string StoredVersion { get; } } public sealed class CultPersistedStoreSnapshot @@ -1881,8 +1882,10 @@ namespace GameCult.Caching /// /// Keys the batch removes that its cache does not hold. Such a removal removes nothing, so it is never in /// ; but a store that holds a record at one of these keys under an id no registered type declares - /// refuses the commit, as it refuses a write of that record. A judges these keys - /// under its lock, with the keys it writes, and never removes them: a record a declared type reads stays where it is. + /// refuses the commit, as it refuses a write of that record. The home's judges + /// these keys under its lock, with the keys it writes; every other attached store judges them first through + /// , each under its own lock, and a refusal anywhere writes nothing. A judged + /// key is never removed: a record a declared type reads stays where it is. /// public IReadOnlyList Checks { get; } public IReadOnlyList<(CultRecordKey Key, string? Version)> Expected { get; } @@ -2659,7 +2662,7 @@ namespace GameCult.Caching : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) : StampFlatten(op.Key)) .ToArray(); - // A removal of a key the cache does not hold removes nothing: it is a check the commit's store judges under its lock. + // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. var checks = batch.Operations.Values .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) .Select(op => op.Key.Value) @@ -2673,13 +2676,15 @@ namespace GameCult.Caching // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) home = only[0].Store; - // Every other store answers the checks now, from what it last loaded: the batch commits through one store. + // Every store the batch does not commit through judges its checks first, each under its own lock, so a refusal or + // contention writes nothing; the home judges its own in CommitBatch. foreach (var (store, _) in _stores) { - if (store == home) + if (store == home || checks.Length == 0) continue; - foreach (var key in checks) - store.RefuseUndeclared(key); + var judged = store.JudgeChecks(checks, wait); + if (judged != CultCommitOutcome.Committed) + return judged; } // What the store holds is what the hold resolved: a variant's delta beside its resolved view. @@ -3577,7 +3582,8 @@ namespace GameCult.Caching /// /// An implementation judges , and its own , under that /// lock with the keys it writes: a key whose record is under an id no registered type declares refuses the commit, and a - /// checked key is never removed. Pass to + /// checked key is never removed. The cache calls on every other store before this commit. Each + /// foreign record a store lists comes from with the version it read, so pass to /// . /// public abstract CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait); @@ -3680,9 +3686,11 @@ namespace GameCult.Caching // The one test of whether a stored record is foreign: no registered type claims its schema id through the catalog it was // stored with. It reads the record's header only, never its payload. Null for a record the registry resolves. - protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog) => + // The one constructor path for a foreign record, so every store kind, one outside this library included, lists it at the + // version it read. + protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog, string storedVersion) => Registry.Claimant(record.SchemaId, catalog, out var persisted) == null - ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt) + ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt, storedVersion) : null; protected void SetForeignRecords(IEnumerable foreign) @@ -3718,7 +3726,8 @@ namespace GameCult.Caching /// judges at its next or as it judges /// : under the lock, with the keys it writes, refused through the same refusal, never /// removed. It calls when that step succeeds, which clears them. A read-only store has no - /// such step and keeps the default. Both shipped store kinds do this. + /// such step and keeps the default. Both shipped store kinds do this. A batch's own checks are judged through + /// . /// /// protected internal virtual void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) @@ -3727,6 +3736,24 @@ namespace GameCult.Caching RefuseUndeclared(key); } + /// + /// Called for a batch's checks on every attached store that is not the batch's home, before the home commits. It judges + /// only the keys it is given: it never writes, stages, clears or calls + /// . + /// + /// + /// The default answers from this store's view () and reads nothing, which is right for a + /// read-only store. A writable store kind that reads its durable state under a lock overrides this to read under that + /// lock and refuse through , returning when it may not wait + /// and the lock is held. A refusal or contention here writes nothing anywhere, because the home has not yet committed. + /// + protected internal virtual CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + foreach (var key in keys) + RefuseUndeclared(key); + return CultCommitOutcome.Committed; + } + // Keys a removal named that this store's cache does not hold, waiting for this store's next flush or commit to judge them. // A refusal takes back the keys it refused; a successful flush or commit clears them. private readonly HashSet _checks = new(StringComparer.Ordinal); @@ -4005,9 +4032,8 @@ namespace GameCult.Caching foreach (var record in snapshot.Records) { var version = versions[record.Key] = durable.Version(record.Key); - if (Foreign(record, snapshot.SchemaCatalog) is { } carried) + if (Foreign(record, snapshot.SchemaCatalog, version) is { } carried) { - carried.StoredVersion = version; foreign.Add(carried); continue; } @@ -4286,6 +4312,22 @@ namespace GameCult.Caching StageChecks(keys); } + // A read-only store answers a batch's checks from its view and never creates the lock file; a writable one reads the file + // under its lock. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + using var fileLock = AcquireLock(wait); + if (fileLock == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares); + return CultCommitOutcome.Committed; + }); + } + // A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id // and the ids it lists compatible. The keys judged are every key staged, written, removed or checked. private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => @@ -4335,9 +4377,8 @@ namespace GameCult.Caching private CultForeignRecord? ForeignAt(Durable? durable, string key) { if (durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is not { } record || - Foreign(record, durable.Snapshot.SchemaCatalog) is not { } carried) + Foreign(record, durable.Snapshot.SchemaCatalog, durable.Version(key)) is not { } carried) return null; - carried.StoredVersion = durable.Version(key); return carried; } @@ -4346,7 +4387,7 @@ namespace GameCult.Caching private CultStoredDocument? DurableAt(Durable? durable, string key) { var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)); - if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog) != null) + if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null) return null; return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _); } From 045db8f98afc2bb3739d881ed6ec4a49f1e9b47f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:07:20 +0200 Subject: [PATCH 10/13] Tests: non-home stores judge checks under their lock, Contended before the home writes, refusal-listed foreign version, outside store versions foreign records Co-Authored-By: Claude Opus 5.5 --- .../GameCult.Caching.ConsumerBuild/Program.cs | 36 ++++++ .../ForeignRecordTests.cs | 117 +++++++++++++++++- 2 files changed, 151 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.ConsumerBuild/Program.cs b/tests/GameCult.Caching.ConsumerBuild/Program.cs index b0f15973..417da1a6 100644 --- a/tests/GameCult.Caching.ConsumerBuild/Program.cs +++ b/tests/GameCult.Caching.ConsumerBuild/Program.cs @@ -5,6 +5,42 @@ using MessagePack; // RefKeyedDictionaryCompilesInAConsumer: this project building is the test. System.Console.WriteLine(new ConsumerFaction().Standing.Count); +// An outside store kind versions its foreign records: this project has no InternalsVisibleTo, so building and running it pins that +// Foreign(record, catalog, version) is all a store kind outside the library needs for ExpectUnchanged to hold over them. +using var outside = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) })); +outside.AddBackingStore(new OutsideStore(), typeof(ConsumerFaction)); +var committed = outside.Commit(batch => +{ + batch.ExpectUnchanged(); + batch.Upsert(new ConsumerFaction { Name = "a" }, new CultRecordHandle(new CultRecordKey("a"))); +}); +if (!committed) + return 1; +return 0; + +// Lists one record under an id no type claims, at version "v1", and commits only when the conditions hold over it. +public sealed class OutsideStore : CacheBackingStore +{ + private static readonly CultSchemaCatalogEntry Entry = new() { SchemaId = "outside.unclaimed", SchemaName = "outside.unclaimed" }; + + public override void PullAll() + { + var record = new CultPersistedRecord { Key = "w", SchemaId = Entry.SchemaId, StoredAt = "t" }; + SetForeignRecords(new[] { Foreign(record, new[] { Entry }, "v1")! }); + } + + public override void Push(CultStoredDocument entry) { } + + public override void Delete(CultStoredDocument entry) { } + + public override void PushAll() { } + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => + request.ConditionsHold(new Dictionary { ["w"] = "v1" }, Entries.Values, ForeignRecords) + ? CultCommitOutcome.Committed + : CultCommitOutcome.Mismatch; +} + [CultDocument("tests.consumer_faction", "tests.consumer_faction.v1")] [MessagePackObject] public sealed class ConsumerFaction diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index a598ed98..6f530b46 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1334,9 +1334,9 @@ namespace GameCult.Caching.Tests Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); } - // A batch that only checks keys commits through the one writable store; every other store answers the checks from its view. + // A batch that only checks keys commits through the one writable store; a read-only store answers the checks from its last load. [Test] - public void AStoreThatIsNotTheHomeAnswersTheChecksFromItsView([Values] bool directory) + public void AReadOnlyStoreAnswersTheChecksFromItsView([Values] bool directory) { var writable = PathOf("home.cc"); using (var seed = Open(writable, Full, directory)) @@ -1396,5 +1396,118 @@ namespace GameCult.Caching.Tests Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); try { cache.Dispose(); } catch (CultSchemaConflictException) { } } + + // Two writable stores: home.cc holds a deck and takes Deck, other.cc holds a gizmo and takes Gizmo, and, when asked, a widget at + // w that no registered type declares, so the cache lists it foreign at the load. + private (CultCache Cache, string Home, string Other) TwoWritableStores(bool directory, bool otherListsWidget) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + { + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + if (otherListsWidget) + seed.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + } + + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + return (cache, home, other); + } + + // Every writable store judges a batch's checks under its own lock, whichever store the batch commits through: a record another + // writer put at a checked key in a store the batch does not write, after the load, refuses the whole commit and writes nothing. + [Test] + public void EveryWritableStoreRefusesACommitThatRemovesItsLateForeignRecord([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + var otherBefore = Fingerprint(other); + + var refusal = Assert.Throws(() => cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + Assert.That(Fingerprint(other), Is.EqualTo(otherBefore)); + Assert.That(Read(home).Records.Select(record => record.Key), Does.Not.Contain("d2")); + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + } + + // The non-home judgement reads the store under its lock, not its listing: a foreign record that store listed at the load, + // since replaced by a declared one, no longer refuses the commit and is not removed. + [Test] + public void ANonHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: true); + using var _ = cache; + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "theirs")), W)); + var theirs = Stored(other, W); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(Stored(other, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } + + // A store that is not the batch's home and cannot take its lock makes the commit contended before the home writes. + [Test] + public void ANonHomeStoreHeldByAnotherWriterMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + { + var outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + } + + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + + // A foreign record a refusal lists carries the version the refusal read, so a conditional commit over the stores that + // list it holds when nothing has changed since. + [Test] + public void ExpectUnchangedHoldsAfterARefusalListedTheForeignRecord([Values] bool directory) + { + var path = PathOf("refusal-listed.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var cache = Open(path, DeckOnly, directory); + using (var late = Open(path, Full, directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + Assert.That(cache.Remove(W), Is.False); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(cache.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + }), Is.True); + } } } From 7be966b4854354e9cefc8a077e659217d07a8208 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:07:21 +0200 Subject: [PATCH 11/13] Contracts: every writable store judges a batch's checks; a store kind lists foreign records at the version it read Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 2 +- .../Contracts/cultcache-store-composition.md | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 69de1c75..7f1d89ba 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,7 +206,7 @@ A record that resolves to no local type is a foreign record, for example one of have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords` in C#), and the cache never holds it. Writes of other records proceed and copy it forward as the file holds it. A write that would replace or remove it is refused with the typed schema conflict -naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. +naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A commit judges such a key in every writable store its cache attaches, each under its own lock: the stores it does not write first, then the store it writes, and a refusal in any of them writes nothing. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 740b5f73..b03627ca 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -265,9 +265,12 @@ refuses to replace a file whose header it cannot read, and leaves it as it was. (cultcache-persistence-format.md, ownership by declaration). - A removal of a key the cache does not hold travels as a check-only key: a commit carries it in the request, a flush stages it, and either store kind - judges it under its lock with the keys it writes, never removing it. A batch + judges it under its lock with the keys it writes, never removing it. Every + writable store judges a batch's checks under its own lock: the stores the + batch does not commit through first (`JudgeChecks`), the home last. A batch that only checks keys writes no file. `ExpectUnchanged` compares the foreign - records a store carries at the versions it read them. + records a store carries at the versions it read them; a store kind lists a + foreign record through `Foreign` with the version it read. - A single-file store holding a variant (the directory store refuses variants) is written only from what its writer last read. A record that moved refuses the whole write with the typed write conflict, and the refused and moved keys From 696a71208823d8c466a392e969d0e4e8c151aac7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:23:14 +0200 Subject: [PATCH 12/13] JudgeChecks (directory): drop the CreateDirectory the commit lease already makes; tests for a never-written non-home store and for a check-less batch beside a held lock Co-Authored-By: Claude Opus 5.5 --- .../DirectoryMessagePackBackingStore.cs | 1 - .../ForeignRecordTests.cs | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index a5e2b18d..eeec3349 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -272,7 +272,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore return base.JudgeChecks(keys, wait); return Held(() => { - Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 6f530b46..c205c54c 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1488,6 +1488,43 @@ namespace GameCult.Caching.Tests Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); } + // A non-home store the cache attached before anything was written to it has no directory and no lease yet: judging a check + // there creates what it locks, finds nothing, and writes no store. + [Test] + public void ANonHomeStoreNothingWasEverWrittenToJudgesAChecksOnlyCommit([Values] bool directory) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("fresh" + Path.DirectorySeparatorChar + "other.cc"); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(File.Exists(other), Is.False, "judging a check writes no store"); + } + + // A batch with no check asks no other store anything: a lock another writer holds on a store the batch does not write does + // not stand in its way. + [Test] + public void ABatchWithoutChecksIsNotHeldUpByAnotherStoreLock() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + Assert.That(cache.TryCommit(batch => batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2"))), Is.EqualTo(CultCommitOutcome.Committed)); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + } + // A foreign record a refusal lists carries the version the refusal read, so a conditional commit over the stores that // list it holds when nothing has changed since. [Test] From b5ef551d843275ab11ed07c29abae0125b6bed7e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 03:14:46 +0200 Subject: [PATCH 13/13] JudgeChecks lock contract: tests pin the read-after-lock order and the directory judge's wait=false A non-home store (single-file and directory) is read only once the commit holds its own lock, so a foreign record landed at a checked key while the commit waits is refused; and a directory judge whose lease is held makes TryCommit Contended at once. Co-Authored-By: Claude Opus 5.5 --- .../ForeignRecordTests.cs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index c205c54c..4c30b9e0 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -1488,6 +1488,85 @@ namespace GameCult.Caching.Tests Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); } + private static void CopyStoreOver(string from, string to, bool directory) + { + if (directory) + { + var source = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(from); + var target = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(to); + Directory.CreateDirectory(target); + foreach (var file in Directory.GetFiles(source, "*.msgpack")) + File.Copy(file, Path.Combine(target, Path.GetFileName(file)), true); + } + File.Copy(from, to, true); + } + + private static FileStream HoldStoreLock(string path, bool directory) => directory + ? new FileStream(Path.Combine(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), ".commit.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None) + : new FileStream(path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + + // A non-home store is judged only after the commit holds that store's own lock: a foreign record another writer lands at the + // checked key while the commit waits for the lock is refused, whichever kind the store is and whether or not the batch writes. + [Test] + public void ANonHomeStoreIsReadOnlyAfterItsLockIsTaken([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + var copy = PathOf("copy.cc"); + CopyStoreOver(other, copy, directory); + using (var late = Open(copy, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + + Exception? thrown = null; + var held = HoldStoreLock(other, directory); + var worker = new System.Threading.Thread(() => + { + try + { + cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + } + catch (Exception ex) { thrown = ex; } + }); + worker.Start(); + // The lock wait is a Thread.Sleep retry loop, so the worker sleeping is the worker waiting for the held lock. + Assert.That(System.Threading.SpinWait.SpinUntil( + () => (worker.ThreadState & System.Threading.ThreadState.WaitSleepJoin) != 0, TimeSpan.FromSeconds(10)), Is.True, "the commit waits on the held lock"); + CopyStoreOver(copy, other, directory); + var otherAfterSwap = Fingerprint(other); + held.Dispose(); + + Assert.That(worker.Join(TimeSpan.FromSeconds(40)), Is.True, "the commit finished"); + Assert.That(thrown, Is.TypeOf()); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore), "the home is unwritten"); + Assert.That(Fingerprint(other), Is.EqualTo(otherAfterSwap), "the non-home store is unwritten"); + } + + // A directory store judging a check honours wait=false: its lease held by another writer makes TryCommit contended at once. + [Test] + public void ADirectoryNonHomeStoreWhoseLeaseIsHeldMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: true, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + CultCommitOutcome outcome; + using (HoldStoreLock(other, directory: true)) + outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + // A non-home store the cache attached before anything was written to it has no directory and no lease yet: judging a check // there creates what it locks, finds nothing, and writes no store. [Test]