From 4e1ba0eb826b097df104c86a62c51daa95bdd473 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 22:30:37 +0000 Subject: [PATCH 1/5] cultcache-py: push_all(soft) skips a readable store, writes an absent one, refuses an unreadable one benchmark.py's InMemoryBackingStore implements the BackingStore protocol and takes the new keyword. Co-Authored-By: Claude Sonnet 5.5 --- .../src/cultcache_py/backing_store.py | 4 +- .../cultcache-py/src/cultcache_py/stores.py | 13 +++-- .../cultcache-py/tests/test_soft_push_all.py | 53 +++++++++++++++++++ .../cultnet-py/src/cultnet_py/benchmark.py | 4 +- 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 packages/cultcache-py/tests/test_soft_push_all.py diff --git a/packages/cultcache-py/src/cultcache_py/backing_store.py b/packages/cultcache-py/src/cultcache_py/backing_store.py index 5a21e496..7a93bbd0 100644 --- a/packages/cultcache-py/src/cultcache_py/backing_store.py +++ b/packages/cultcache-py/src/cultcache_py/backing_store.py @@ -67,10 +67,12 @@ class BackingStore(Protocol): def delete(self, type: str, key: str) -> None: ... - def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: existing = { (envelope.type, envelope.key): envelope for envelope in self.pull_all() } + if soft and existing: + return for envelope in envelopes: existing[(envelope.type, envelope.key)] = envelope self._replace_all(list(existing.values())) diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index c15ac32e..35b53af3 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -106,9 +106,11 @@ class JsonLinesBackingStore: existing[(envelope.type, envelope.key)] = envelope self._replace_all(list(existing.values())) - def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: with self._lock: existing = {(item.type, item.key): item for item in self.pull_all()} + if soft and self.path.exists(): + return for envelope in envelopes: existing[(envelope.type, envelope.key)] = envelope self._replace_all(list(existing.values())) @@ -163,9 +165,14 @@ class SingleFileMessagePackBackingStore: with self._write_transaction(): self._replace_all([envelope], self.pull_all()) - def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: + """A soft push writes only where nothing is at the path: a store the reader reads is left byte for byte, and a file it + cannot read is refused by the same read, as in TS and Rust.""" with self._write_transaction(): - self._replace_all(list(envelopes), self.pull_all()) + arrived = self.pull_all() + if soft and self.path.exists(): + return + self._replace_all(list(envelopes), arrived) def delete(self, type: str, key: str) -> None: with self._write_transaction(): diff --git a/packages/cultcache-py/tests/test_soft_push_all.py b/packages/cultcache-py/tests/test_soft_push_all.py new file mode 100644 index 00000000..a1874c7c --- /dev/null +++ b/packages/cultcache-py/tests/test_soft_push_all.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path + +from cultcache_py import JsonLinesBackingStore, SingleFileMessagePackBackingStore, StoreUnreadableError +from cultcache_py.backing_store import CultCacheEnvelope + + +def envelope(key: str) -> CultCacheEnvelope: + return CultCacheEnvelope(key=key, type="t", payload=b"\x01", stored_at="2026-11-06T00:00:00Z") + + +class SoftPushAllTests(unittest.TestCase): + """A soft push writes where nothing is, skips a store it reads, and refuses one it cannot read (TS and Rust parity).""" + + def setUp(self) -> None: + self.dir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.path = self.dir / "state.cc" + + def test_over_a_readable_store_it_leaves_the_file_byte_for_byte(self) -> None: + for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore): + with self.subTest(make.__name__): + path = self.dir / make.__name__ + store = make(path) + store.push_all([envelope("a")]) + before = path.read_bytes() + store.push_all([envelope("b")], soft=True) + self.assertEqual(before, path.read_bytes()) + self.assertEqual(["a"], [item.key for item in make(path).pull_all()]) + store.push_all([envelope("b")]) # a plain push still lands what the caller holds + self.assertEqual(["a", "b"], sorted(item.key for item in make(path).pull_all())) + + def test_over_an_absent_path_it_writes(self) -> None: + for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore): + with self.subTest(make.__name__): + path = self.dir / make.__name__ + make(path).push_all([envelope("a")], soft=True) + self.assertEqual(["a"], [item.key for item in make(path).pull_all()]) + + def test_over_an_unreadable_file_it_refuses_and_leaves_it(self) -> None: + junk = b"\xc1\x01\x02\x03" + self.path.write_bytes(junk) + with self.assertRaises(StoreUnreadableError): + SingleFileMessagePackBackingStore(self.path).push_all([envelope("a")], soft=True) + self.assertEqual(junk, self.path.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/src/cultnet_py/benchmark.py b/packages/cultnet-py/src/cultnet_py/benchmark.py index 83834635..0793bfc7 100644 --- a/packages/cultnet-py/src/cultnet_py/benchmark.py +++ b/packages/cultnet-py/src/cultnet_py/benchmark.py @@ -137,7 +137,9 @@ class InMemoryBackingStore: def push(self, envelope: CultCacheEnvelope) -> None: self._records[(envelope.type, envelope.key)] = envelope - def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: + if soft and self._records: + return for envelope in envelopes: self.push(envelope) From 0a1486eccd24756f6754f3b8d96ea782fd354cd9 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 22:33:01 +0000 Subject: [PATCH 2/5] C#: soft flush skips a readable store and keeps staged writes dirty; Mesh and CultNetLocal pass the flag CacheBackingStore.PushAll(bool soft) is the one abstract owner; PushAll() stays as the plain call. Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 5 +- src/GameCult.Caching/CultCache.cs | 22 ++++-- src/GameCult.Mesh/CultMesh.cs | 2 +- src/GameCult.Networking/CultNetLocal.cs | 2 +- .../GameCult.Caching.ConsumerBuild/Program.cs | 2 +- .../OutsideStoreJudgeTests.cs | 2 +- .../BackingStoreTests.cs | 76 +++++++++++++++++++ .../CultCacheJournalTests.cs | 2 +- .../CultCacheObserverTests.cs | 2 +- .../StoreRoutingTests.cs | 4 +- 10 files changed, 103 insertions(+), 16 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 8f76cbae..d16907b6 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -237,7 +237,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore }); } - public override void PushAll() + public override void PushAll(bool soft) { ThrowIfReadOnly(); Held(() => @@ -248,6 +248,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); + // A soft flush writes only where no manifest is; a manifest ReadManifest read stays as it is, its staged writes staged. + if (soft && File.Exists(_manifestFile.FullName)) + return; RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty(), Array.Empty()); if (IsDirty) WriteGeneration(manifest, catalogBytes); diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 48bad3a3..91dfa9ac 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2537,19 +2537,19 @@ namespace GameCult.Caching return Task.CompletedTask; } - public void FlushAllBackingStores() => Held(() => + public void FlushAllBackingStores(bool soft = false) => Held(() => { foreach (var (store, _) in _stores) { if (!store.IsReadOnly && (store.IsDirty || store.HasStagedChecks)) - store.PushAll(); + store.PushAll(soft); } return true; }); - public Task FlushAsync() + public Task FlushAsync(bool soft = false) { - FlushAllBackingStores(); + FlushAllBackingStores(soft); return Task.CompletedTask; } @@ -3945,7 +3945,13 @@ namespace GameCult.Caching public abstract void PullAll(); public abstract void Push(CultStoredDocument entry); public abstract void Delete(CultStoredDocument entry); - public abstract void PushAll(); + public void PushAll() => PushAll(soft: false); + + /// + /// Applies the staged writes. A soft flush writes only where nothing is at the path: over a store it reads it returns without + /// writing and leaves the staged writes staged and dirty for a later plain flush, and over a file it cannot read it refuses. + /// + public abstract void PushAll(bool soft); /// /// One durable step under the store's exclusive lock: the batch and every staged write land onto the store as it is then, @@ -4184,7 +4190,7 @@ namespace GameCult.Caching /// /// /// A writable store kind that reads its durable state under a lock overrides this to call , and - /// judges at its next or as it judges + /// judges at its next or as it judges /// : under the lock, with the keys it writes, refused through the same refusal, never /// removed. It calls when that step succeeds, which clears them. A read-only store has no /// such step and keeps the default. Both shipped store kinds do this. A batch's own checks are judged through @@ -4661,7 +4667,7 @@ namespace GameCult.Caching // only per staged key. A key whose durable record is under an id this cache does not declare refuses the write, and in a store // holding a variant it lands only onto the store as this cache last read it (both ApplyWriteSet). // A file it cannot read (truncated, trailing bytes, not a store, a format it does not read) is refused and left as it is. - public override void PushAll() + public override void PushAll(bool soft) { ThrowIfReadOnly(); Held(() => @@ -4669,6 +4675,8 @@ namespace GameCult.Caching using (AcquireLock(wait: true)) { var disk = ReadSnapshot(); + if (soft && disk != null) + return; var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); Wrote(ApplyWriteSet(disk, upserts, removals, Array.Empty())); } diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index e3f23430..6ea48e0d 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -432,7 +432,7 @@ namespace GameCult.Mesh /// public Task FlushAsync(bool soft = false) { - return _host.FlushAsync(); + return _host.FlushAsync(soft); } /// diff --git a/src/GameCult.Networking/CultNetLocal.cs b/src/GameCult.Networking/CultNetLocal.cs index 87942ce8..e884cd8d 100644 --- a/src/GameCult.Networking/CultNetLocal.cs +++ b/src/GameCult.Networking/CultNetLocal.cs @@ -100,7 +100,7 @@ namespace GameCult.Networking /// public Task FlushAsync(bool soft = false) { - return Cache.FlushAsync(); + return Cache.FlushAsync(soft); } /// diff --git a/tests/GameCult.Caching.ConsumerBuild/Program.cs b/tests/GameCult.Caching.ConsumerBuild/Program.cs index 60376068..f09edb8c 100644 --- a/tests/GameCult.Caching.ConsumerBuild/Program.cs +++ b/tests/GameCult.Caching.ConsumerBuild/Program.cs @@ -56,7 +56,7 @@ public sealed class OutsideStore : CacheBackingStore public override void Delete(CultStoredDocument entry) { } - public override void PushAll() { } + public override void PushAll(bool soft) { } public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => request.ConditionsHold(new Dictionary { ["w"] = "v1" }, Entries.Values, ForeignRecords) diff --git a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs index e47e4d5f..bd2fe047 100644 --- a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs +++ b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs @@ -136,7 +136,7 @@ namespace GameCult.Caching.OutsideStoreTests IsDirty = true; }); - public override void PushAll() => Held(() => Apply(Array.Empty(), Array.Empty(), Array.Empty())); + public override void PushAll(bool soft) => Held(() => Apply(Array.Empty(), Array.Empty(), Array.Empty())); public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() => { diff --git a/tests/GameCult.Caching.Tests/BackingStoreTests.cs b/tests/GameCult.Caching.Tests/BackingStoreTests.cs index 7e88c283..ffc869e6 100644 --- a/tests/GameCult.Caching.Tests/BackingStoreTests.cs +++ b/tests/GameCult.Caching.Tests/BackingStoreTests.cs @@ -56,6 +56,82 @@ namespace GameCult.Caching.Tests } } + [Test] + public async Task SoftFlush_Over_Readable_Store_Leaves_It_Byte_For_Byte_And_Keeps_Staged_Writes_Dirty() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + try + { + var store = new SingleFileMessagePackBackingStore(filePath); + var cache = new CultCache(); + cache.AddBackingStore(store); + await cache.AddAsync(new NamedTestEntry { Name = "first", Value = "one" }); + await cache.FlushAsync(); + var before = File.ReadAllBytes(filePath); + + var staged = await cache.AddAsync(new NamedTestEntry { Name = "second", Value = "two" }); + await cache.FlushAsync(soft: true); + + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(before), "a soft flush rewrote a store it reads"); + Assert.That(store.IsDirty, Is.True, "a skipped soft flush discarded a staged write"); + + await cache.FlushAsync(); + var reader = new CultCache(); + reader.AddBackingStore(new SingleFileMessagePackBackingStore(filePath)); + await reader.PullAllBackingStoresAsync(); + Assert.That(reader.Get(staged.Key)!.Value, Is.EqualTo("two")); + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + [Test] + public async Task SoftFlush_Over_Absent_Path_Writes() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + try + { + var store = new SingleFileMessagePackBackingStore(filePath); + var cache = new CultCache(); + cache.AddBackingStore(store); + await cache.AddAsync(new NamedTestEntry { Name = "first", Value = "one" }); + + await cache.FlushAsync(soft: true); + + Assert.That(File.Exists(filePath), Is.True); + Assert.That(store.IsDirty, Is.False); + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + [Test] + public async Task SoftFlush_Over_Unreadable_File_Refuses_And_Leaves_It() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + try + { + var junk = new byte[] { 0xC1, 0x01, 0x02, 0x03 }; + var store = new SingleFileMessagePackBackingStore(filePath); + var cache = new CultCache(); + cache.AddBackingStore(store); + await cache.AddAsync(new NamedTestEntry { Name = "first", Value = "one" }); + File.WriteAllBytes(filePath, junk); + + Assert.That(() => store.PushAll(soft: true), Throws.InstanceOf()); + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(junk)); + Assert.That(store.IsDirty, Is.True); + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + [Test] public async Task SingleFileMessagePackBackingStore_PullAll_SeesFileCreatedAfterReaderOpened() { diff --git a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs index 404a75cb..f4009a6c 100644 --- a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs @@ -338,7 +338,7 @@ namespace GameCult.Caching.Tests public override void Push(CultStoredDocument entry) { } public override void Delete(CultStoredDocument entry) { } - public override void PushAll() { } + public override void PushAll(bool soft) { } public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => CultCommitOutcome.Committed; } diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs index b2c8e7f5..92861214 100644 --- a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -180,7 +180,7 @@ namespace GameCult.Caching.Tests public override void Push(CultStoredDocument entry) { } public override void Delete(CultStoredDocument entry) { } - public override void PushAll() { } + public override void PushAll(bool soft) { } public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => CultCommitOutcome.Committed; } diff --git a/tests/GameCult.Caching.Tests/StoreRoutingTests.cs b/tests/GameCult.Caching.Tests/StoreRoutingTests.cs index af7e7da7..3ccd7a04 100644 --- a/tests/GameCult.Caching.Tests/StoreRoutingTests.cs +++ b/tests/GameCult.Caching.Tests/StoreRoutingTests.cs @@ -699,10 +699,10 @@ namespace GameCult.Caching.Tests _nested = nested; } - public override void PushAll() + public override void PushAll(bool soft) { _nested(); - base.PushAll(); + base.PushAll(soft); } } From fb80e327494184bfddfbb0f62f8c6931daaf0a27 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 22:44:01 +0000 Subject: [PATCH 3/5] Soft flush: tests for the survivors of the hand mutations (directory store, JSONL and protocol default, host and node passthrough, plain PushAll); revert the benchmark.py touch The InMemoryBackingStore in cultnet-py benchmark.py has no soft caller (the Python cache never passes soft), so the keyword there was unowned. Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-py/tests/test_soft_push_all.py | 28 +++++++++- .../cultnet-py/src/cultnet_py/benchmark.py | 4 +- .../BackingStoreTests.cs | 25 +++++++++ .../DirectoryStoreDurabilityTests.cs | 48 +++++++++++++++++ .../NetworkingTests.cs | 53 +++++++++++++++++++ 5 files changed, 154 insertions(+), 4 deletions(-) diff --git a/packages/cultcache-py/tests/test_soft_push_all.py b/packages/cultcache-py/tests/test_soft_push_all.py index a1874c7c..4bf8d4c1 100644 --- a/packages/cultcache-py/tests/test_soft_push_all.py +++ b/packages/cultcache-py/tests/test_soft_push_all.py @@ -6,7 +6,7 @@ import unittest from pathlib import Path from cultcache_py import JsonLinesBackingStore, SingleFileMessagePackBackingStore, StoreUnreadableError -from cultcache_py.backing_store import CultCacheEnvelope +from cultcache_py.backing_store import BackingStore, CultCacheEnvelope def envelope(key: str) -> CultCacheEnvelope: @@ -48,6 +48,32 @@ class SoftPushAllTests(unittest.TestCase): SingleFileMessagePackBackingStore(self.path).push_all([envelope("a")], soft=True) self.assertEqual(junk, self.path.read_bytes()) + def test_over_an_unreadable_jsonl_file_it_refuses_and_leaves_it(self) -> None: + junk = b"this is not json\n" + self.path.write_bytes(junk) + with self.assertRaises(ValueError): + JsonLinesBackingStore(self.path).push_all([envelope("a")], soft=True) + self.assertEqual(junk, self.path.read_bytes()) + + def test_the_protocol_default_push_all_has_the_same_three_branches(self) -> None: + class Memory(BackingStore): + def __init__(self) -> None: + self.items: list[CultCacheEnvelope] = [] + + def pull_all(self) -> list[CultCacheEnvelope]: + return list(self.items) + + def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + self.items = list(envelopes) + + store = Memory() + store.push_all([envelope("a")], soft=True) # nothing there: writes + self.assertEqual(["a"], [item.key for item in store.items]) + store.push_all([envelope("b")], soft=True) # something there: skips + self.assertEqual(["a"], [item.key for item in store.items]) + store.push_all([envelope("b")]) # plain: lands + self.assertEqual(["a", "b"], sorted(item.key for item in store.items)) + if __name__ == "__main__": unittest.main() diff --git a/packages/cultnet-py/src/cultnet_py/benchmark.py b/packages/cultnet-py/src/cultnet_py/benchmark.py index 0793bfc7..83834635 100644 --- a/packages/cultnet-py/src/cultnet_py/benchmark.py +++ b/packages/cultnet-py/src/cultnet_py/benchmark.py @@ -137,9 +137,7 @@ class InMemoryBackingStore: def push(self, envelope: CultCacheEnvelope) -> None: self._records[(envelope.type, envelope.key)] = envelope - def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: - if soft and self._records: - return + def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: for envelope in envelopes: self.push(envelope) diff --git a/tests/GameCult.Caching.Tests/BackingStoreTests.cs b/tests/GameCult.Caching.Tests/BackingStoreTests.cs index ffc869e6..e6cd5dc8 100644 --- a/tests/GameCult.Caching.Tests/BackingStoreTests.cs +++ b/tests/GameCult.Caching.Tests/BackingStoreTests.cs @@ -87,6 +87,31 @@ namespace GameCult.Caching.Tests } } + [Test] + public async Task PlainPushAll_Over_Readable_Store_Writes_The_Staged_Records() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + try + { + var store = new SingleFileMessagePackBackingStore(filePath); + var cache = new CultCache(); + cache.AddBackingStore(store); + await cache.AddAsync(new NamedTestEntry { Name = "first", Value = "one" }); + store.PushAll(); + var before = File.ReadAllBytes(filePath); + + await cache.AddAsync(new NamedTestEntry { Name = "second", Value = "two" }); + store.PushAll(); + + Assert.That(File.ReadAllBytes(filePath), Is.Not.EqualTo(before), "a plain PushAll skipped a store it reads"); + Assert.That(store.IsDirty, Is.False); + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + [Test] public async Task SoftFlush_Over_Absent_Path_Writes() { diff --git a/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs b/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs index 840110bc..41af7c1b 100644 --- a/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs +++ b/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs @@ -208,6 +208,54 @@ namespace GameCult.Caching.Tests Assert.That(reader.AllEntries, Is.Empty); } + [Test] + public async Task SoftFlushOverAbsentManifestWrites() + { + using var cache = Open(out var store); + var key = await cache.UpsertAsync(typeof(Page), new Page { Name = "a", Text = "one" }); + + cache.FlushAllBackingStores(soft: true); + + Assert.That(File.Exists(_manifest), Is.True); + Assert.That(store.IsDirty, Is.False); + using var reader = Open(out _); + Assert.That(reader.Get(key)?.Text, Is.EqualTo("one")); + } + + [Test] + public async Task SoftFlushOverReadableManifestLeavesItAndKeepsStagedWritesDirty() + { + using var cache = Open(out var store); + await cache.UpsertAsync(typeof(Page), new Page { Name = "a", Text = "one" }); + cache.FlushAllBackingStores(); + var before = File.ReadAllBytes(_manifest); + var pages = Directory.GetFiles(_records, "*.msgpack").OrderBy(path => path).ToArray(); + var second = await cache.UpsertAsync(typeof(Page), new Page { Name = "b", Text = "two" }); + + cache.FlushAllBackingStores(soft: true); + + Assert.That(File.ReadAllBytes(_manifest), Is.EqualTo(before), "a soft flush rewrote a manifest it reads"); + Assert.That(Directory.GetFiles(_records, "*.msgpack").OrderBy(path => path).ToArray(), Is.EqualTo(pages)); + Assert.That(store.IsDirty, Is.True, "a skipped soft flush discarded a staged write"); + + cache.FlushAllBackingStores(); + using var reader = Open(out _); + Assert.That(reader.Get(second)?.Text, Is.EqualTo("two")); + } + + [Test] + public async Task SoftFlushOverUnreadableManifestRefusesAndLeavesIt() + { + using var cache = Open(out var store); + await cache.UpsertAsync(typeof(Page), new Page { Name = "a", Text = "one" }); + var junk = new byte[] { 0xC1, 0x01, 0x02, 0x03 }; + File.WriteAllBytes(_manifest, junk); + + Assert.That(() => store.PushAll(soft: true), Throws.Exception); + Assert.That(File.ReadAllBytes(_manifest), Is.EqualTo(junk)); + Assert.That(store.IsDirty, Is.True); + } + private CultCache Open(out DirectoryMessagePackBackingStore store) { var cache = new CultCache(Registry); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 606d5625..09764518 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -7377,6 +7377,59 @@ namespace GameCult.Networking.Tests Assert.That(developmentServer.LogSensitivePayloads, Is.True); } + [Test] + public async Task CultNetLocal_Host_SoftFlush_Passes_The_Flag_To_The_Store() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultnet-host-soft-{Guid.NewGuid():N}.msgpack"); + CultNetHost? host = null; + + try + { + host = await CultNetLocal.CreateHostAsync(filePath, new CultNetHostOptions { StartServer = false }); + await host.Cache.UpsertAsync(new PlayerData { PlayerId = Guid.NewGuid(), Email = "a@example.test", PasswordHash = "hash", Username = "First" }); + await host.FlushAsync(); + var before = File.ReadAllBytes(filePath); + + await host.Cache.UpsertAsync(new PlayerData { PlayerId = Guid.NewGuid(), Email = "b@example.test", PasswordHash = "hash", Username = "Second" }); + await host.FlushAsync(soft: true); + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(before), "a soft host flush rewrote a store it reads"); + + await host.FlushAsync(); + Assert.That(File.ReadAllBytes(filePath), Is.Not.EqualTo(before), "a plain host flush after a skipped soft flush did not land the staged write"); + } + finally + { + host?.Dispose(); + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + [Test] + public async Task CultMeshNode_SoftFlush_Passes_The_Flag_To_The_Store() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "cultmesh-soft-flush", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + var cachePath = Path.Combine(rootPath, "world.ccmp"); + var documents = new CultNetDocumentRegistry() + .Register(CultNetDocumentBinding.ForDocument()); + + using var node = await CultMesh.CreateNodeAsync(cachePath, new CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = new CultNetDatabaseOptions { RuntimeId = "soft-flush-local", DocumentRegistry = documents } + }); + await node.Database.PutAsync(new CultRecordKey("note:one"), new MeshQuickstartNote { NoteId = "note:one", Body = "one" }); + await node.FlushAsync(); + var before = File.ReadAllBytes(cachePath); + + await node.Database.PutAsync(new CultRecordKey("note:two"), new MeshQuickstartNote { NoteId = "note:two", Body = "two" }); + await node.FlushAsync(soft: true); + Assert.That(File.ReadAllBytes(cachePath), Is.EqualTo(before), "a soft node flush rewrote a store it reads"); + + await node.FlushAsync(); + Assert.That(File.ReadAllBytes(cachePath), Is.Not.EqualTo(before), "a plain node flush after a skipped soft flush did not land the staged write"); + } + [Test] public async Task CultNetLocal_CreateHostAsync_Wires_Durable_Cache_Without_Starting_Server() { From 80eed16c0d1c00dfa899d7ccab4f3e57c3fbcb46 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 23:13:57 +0000 Subject: [PATCH 4/5] Soft flush: the reader's own answer decides (no second stat, protocol default included); pin the readable-empty store, spy the flag through Mesh and host, changelog the breaking PushAll Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-py/CHANGELOG.md | 1 + .../src/cultcache_py/backing_store.py | 9 +-- .../cultcache-py/src/cultcache_py/stores.py | 23 ++++--- .../cultcache-py/tests/test_soft_push_all.py | 39 ++++++++++-- .../DirectoryMessagePackBackingStore.cs | 8 ++- .../BackingStoreTests.cs | 30 ++++++++++ .../DirectoryStoreDurabilityTests.cs | 18 ++++++ .../NetworkingTests.cs | 60 +++++++++++++++++++ unity/org.gamecult.cultlib/CHANGELOG.md | 1 + 9 files changed, 170 insertions(+), 19 deletions(-) diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index ea3f72fd..e44c2430 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -13,6 +13,7 @@ ### Breaking +- `push_all(envelopes, soft=False)` on the single-file and JSONL stores and on the `BackingStore` protocol takes the soft flag, as TypeScript, Rust and C# do: a soft push writes only where nothing is at the path; over a store the reader reads, empty included, it returns without writing, and over a file the reader cannot read it refuses (`StoreUnreadableError` for the single-file store). The `BackingStore` protocol's default `push_all` cannot tell an absent store from an empty one (a pull returns a list), so a soft push through it always skips a store whose pull returns; a store kind that can tell nothing-at-the-path overrides `push_all`. A store kind that overrides `push_all(envelopes)` without the keyword fails when a caller passes `soft`. - A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`. - A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` ` payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#. - A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it. diff --git a/packages/cultcache-py/src/cultcache_py/backing_store.py b/packages/cultcache-py/src/cultcache_py/backing_store.py index 7a93bbd0..7b07e7be 100644 --- a/packages/cultcache-py/src/cultcache_py/backing_store.py +++ b/packages/cultcache-py/src/cultcache_py/backing_store.py @@ -68,11 +68,12 @@ class BackingStore(Protocol): ... def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: - existing = { - (envelope.type, envelope.key): envelope for envelope in self.pull_all() - } - if soft and existing: + # A pull that returns is a store the reader read, empty included: the protocol cannot say "absent", so a soft push + # through this default always skips. A store that can tell nothing-at-the-path overrides push_all. + pulled = self.pull_all() + if soft: return + existing = {(envelope.type, envelope.key): envelope for envelope in pulled} for envelope in envelopes: existing[(envelope.type, envelope.key)] = envelope self._replace_all(list(existing.values())) diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 35b53af3..2be88200 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -76,10 +76,14 @@ class JsonLinesBackingStore: self._lock = threading.RLock() def pull_all(self) -> list[CultCacheEnvelope]: + return self._pull() or [] + + def _pull(self) -> list[CultCacheEnvelope] | None: + """The envelopes the reader reads, or None when nothing is at the path.""" with self._lock: text = _read_store(self.path, lambda path: path.read_text(encoding="utf-8")) if text is None: - return [] + return None envelopes: list[CultCacheEnvelope] = [] for line_number, line in enumerate(text.splitlines(), start=1): if not line.strip(): @@ -108,9 +112,10 @@ class JsonLinesBackingStore: def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - if soft and self.path.exists(): + arrived = self._pull() + if soft and arrived is not None: return + existing = {(item.type, item.key): item for item in arrived or []} for envelope in envelopes: existing[(envelope.type, envelope.key)] = envelope self._replace_all(list(existing.values())) @@ -144,13 +149,17 @@ class SingleFileMessagePackBackingStore: self._format = STORE_FORMAT_VERSION def pull_all(self) -> list[CultCacheEnvelope]: + return self._pull() or [] + + def _pull(self) -> list[CultCacheEnvelope] | None: + """The envelopes the reader reads, or None when nothing is at the path.""" with self._lock: # An empty file is not a store (no writer leaves one), so it is decoded and refused. # The disk decides the header: a file that is gone is not marked. self._format = STORE_FORMAT_VERSION data = _read_store(self.path, Path.read_bytes) if data is None: - return [] + return None try: decoded = msgpack.unpackb(data, raw=False) snapshot = _decode_snapshot(decoded) @@ -169,10 +178,10 @@ class SingleFileMessagePackBackingStore: """A soft push writes only where nothing is at the path: a store the reader reads is left byte for byte, and a file it cannot read is refused by the same read, as in TS and Rust.""" with self._write_transaction(): - arrived = self.pull_all() - if soft and self.path.exists(): + arrived = self._pull() + if soft and arrived is not None: return - self._replace_all(list(envelopes), arrived) + self._replace_all(list(envelopes), arrived or []) def delete(self, type: str, key: str) -> None: with self._write_transaction(): diff --git a/packages/cultcache-py/tests/test_soft_push_all.py b/packages/cultcache-py/tests/test_soft_push_all.py index 4bf8d4c1..9a2c6e03 100644 --- a/packages/cultcache-py/tests/test_soft_push_all.py +++ b/packages/cultcache-py/tests/test_soft_push_all.py @@ -34,6 +34,20 @@ class SoftPushAllTests(unittest.TestCase): store.push_all([envelope("b")]) # a plain push still lands what the caller holds self.assertEqual(["a", "b"], sorted(item.key for item in make(path).pull_all())) + def test_over_a_readable_store_holding_no_records_it_leaves_the_file_byte_for_byte(self) -> None: + for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore): + with self.subTest(make.__name__): + path = self.dir / make.__name__ + store = make(path) + store.push_all([envelope("a")]) + store.delete("t", "a") + self.assertTrue(path.exists()) + self.assertEqual([], make(path).pull_all()) + before = path.read_bytes() + store.push_all([envelope("b")], soft=True) + self.assertEqual(before, path.read_bytes()) + self.assertEqual([], make(path).pull_all()) + def test_over_an_absent_path_it_writes(self) -> None: for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore): with self.subTest(make.__name__): @@ -55,7 +69,7 @@ class SoftPushAllTests(unittest.TestCase): JsonLinesBackingStore(self.path).push_all([envelope("a")], soft=True) self.assertEqual(junk, self.path.read_bytes()) - def test_the_protocol_default_push_all_has_the_same_three_branches(self) -> None: + def test_the_protocol_default_push_all_skips_what_its_pull_read_empty_included(self) -> None: class Memory(BackingStore): def __init__(self) -> None: self.items: list[CultCacheEnvelope] = [] @@ -67,13 +81,28 @@ class SoftPushAllTests(unittest.TestCase): self.items = list(envelopes) store = Memory() - store.push_all([envelope("a")], soft=True) # nothing there: writes + store.push_all([envelope("a")], soft=True) # the pull read an empty store: skips + self.assertEqual([], store.items) + store.push_all([envelope("a")]) # plain: lands + store.push_all([envelope("b")], soft=True) # a store with records: skips self.assertEqual(["a"], [item.key for item in store.items]) - store.push_all([envelope("b")], soft=True) # something there: skips - self.assertEqual(["a"], [item.key for item in store.items]) - store.push_all([envelope("b")]) # plain: lands + store.push_all([envelope("b")]) self.assertEqual(["a", "b"], sorted(item.key for item in store.items)) + def test_the_protocol_default_push_all_refuses_where_its_pull_refuses(self) -> None: + class Unreadable(BackingStore): + replaced = False + + def pull_all(self) -> list[CultCacheEnvelope]: + raise StoreUnreadableError(Path("x"), ValueError("junk")) + + def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + Unreadable.replaced = True + + with self.assertRaises(StoreUnreadableError): + Unreadable().push_all([envelope("a")], soft=True) + self.assertFalse(Unreadable.replaced) + if __name__ == "__main__": unittest.main() diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index d16907b6..5005b995 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -247,9 +247,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); - var (manifest, catalogBytes) = ReadManifest(); - // A soft flush writes only where no manifest is; a manifest ReadManifest read stays as it is, its staged writes staged. - if (soft && File.Exists(_manifestFile.FullName)) + var manifestBytes = ReadManifestBytes(); + var (manifest, catalogBytes) = ParseManifest(manifestBytes); + // A soft flush writes only where no manifest is: the reader's answer (null is absent), not a second look at the path. + // A manifest it read stays as it is, its staged writes staged. + if (soft && manifestBytes != null) return; RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty(), Array.Empty()); if (IsDirty) diff --git a/tests/GameCult.Caching.Tests/BackingStoreTests.cs b/tests/GameCult.Caching.Tests/BackingStoreTests.cs index e6cd5dc8..8d5c3424 100644 --- a/tests/GameCult.Caching.Tests/BackingStoreTests.cs +++ b/tests/GameCult.Caching.Tests/BackingStoreTests.cs @@ -87,6 +87,36 @@ namespace GameCult.Caching.Tests } } + [Test] + public async Task SoftFlush_Over_Readable_Store_Holding_No_Records_Leaves_It() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + try + { + var store = new SingleFileMessagePackBackingStore(filePath); + var cache = new CultCache(); + cache.AddBackingStore(store); + var first = await cache.AddAsync(new NamedTestEntry { Name = "first", Value = "one" }); + await cache.FlushAsync(); + cache.Remove(first.Key); + await cache.FlushAsync(); + var before = File.ReadAllBytes(filePath); + var empty = new CultCache(); + empty.AddBackingStore(new SingleFileMessagePackBackingStore(filePath)); + Assert.That(empty.AllEntries, Is.Empty, "the store is not readable-empty"); + + await cache.AddAsync(new NamedTestEntry { Name = "second", Value = "two" }); + await cache.FlushAsync(soft: true); + + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(before), "a soft flush wrote over a readable store that holds no records"); + Assert.That(store.IsDirty, Is.True); + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + [Test] public async Task PlainPushAll_Over_Readable_Store_Writes_The_Staged_Records() { diff --git a/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs b/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs index 41af7c1b..942ae891 100644 --- a/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs +++ b/tests/GameCult.Caching.Tests/DirectoryStoreDurabilityTests.cs @@ -243,6 +243,24 @@ namespace GameCult.Caching.Tests Assert.That(reader.Get(second)?.Text, Is.EqualTo("two")); } + [Test] + public async Task SoftFlushOverReadableEmptyManifestLeavesIt() + { + using var cache = Open(out var store); + var key = await cache.UpsertAsync(typeof(Page), new Page { Name = "a", Text = "one" }); + cache.FlushAllBackingStores(); + cache.Remove(key); + cache.FlushAllBackingStores(); + var before = File.ReadAllBytes(_manifest); + using (var reader = Open(out _)) Assert.That(reader.AllEntries, Is.Empty, "the store is not readable-empty"); + await cache.UpsertAsync(typeof(Page), new Page { Name = "b", Text = "two" }); + + cache.FlushAllBackingStores(soft: true); + + Assert.That(File.ReadAllBytes(_manifest), Is.EqualTo(before), "a soft flush wrote over a readable store that holds no records"); + Assert.That(store.IsDirty, Is.True); + } + [Test] public async Task SoftFlushOverUnreadableManifestRefusesAndLeavesIt() { diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 09764518..b566dca3 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -7377,6 +7377,66 @@ namespace GameCult.Networking.Tests Assert.That(developmentServer.LogSensitivePayloads, Is.True); } + // Records the flag each PushAll took, and holds nothing: a flush that decided soft itself and never asked the store records nothing. + private sealed class SoftFlagSpyStore : CacheBackingStore + { + public sealed class SpyHome { } + + public readonly List Flags = new(); + public void Stage() => IsDirty = true; + public override void PullAll() { } + public override void Push(CultStoredDocument entry) { } + public override void Delete(CultStoredDocument entry) { } + public override void PushAll(bool soft) => Flags.Add(soft); + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => CultCommitOutcome.Committed; + } + + [Test] + public async Task CultNetLocal_Host_FlushAsync_Hands_The_Store_The_Flag_It_Was_Given() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultnet-host-spy-{Guid.NewGuid():N}.msgpack"); + CultNetHost? host = null; + try + { + host = await CultNetLocal.CreateHostAsync(filePath, new CultNetHostOptions { StartServer = false }); + var spy = new SoftFlagSpyStore(); + host.Cache.AddBackingStore(spy, typeof(SoftFlagSpyStore.SpyHome)); + spy.Stage(); + + await host.FlushAsync(soft: true); + await host.FlushAsync(); + + Assert.That(spy.Flags, Is.EqualTo(new[] { true, false })); + } + finally + { + host?.Dispose(); + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + [Test] + public async Task CultMeshNode_FlushAsync_Hands_The_Store_The_Flag_It_Was_Given() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "cultmesh-soft-spy", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + var documents = new CultNetDocumentRegistry() + .Register(CultNetDocumentBinding.ForDocument()); + using var node = await CultMesh.CreateNodeAsync(Path.Combine(rootPath, "world.ccmp"), new CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = new CultNetDatabaseOptions { RuntimeId = "soft-spy-local", DocumentRegistry = documents } + }); + var spy = new SoftFlagSpyStore(); + node.Cache.AddBackingStore(spy, typeof(SoftFlagSpyStore.SpyHome)); + spy.Stage(); + + await node.FlushAsync(soft: true); + await node.FlushAsync(); + + Assert.That(spy.Flags, Is.EqualTo(new[] { true, false })); + } + [Test] public async Task CultNetLocal_Host_SoftFlush_Passes_The_Flag_To_The_Store() { diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index a4a2bc11..612aee04 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this package are documented in this file. ### Breaking +- `CacheBackingStore.PushAll()` is no longer abstract: it is a non-virtual convenience over the new abstract `PushAll(bool soft)`. A store kind that overrides `PushAll()` no longer compiles; override `PushAll(bool soft)` instead. A soft flush writes only where nothing is at the path: over a store the reader reads, empty included, it returns without writing and leaves the staged writes staged and dirty for a later plain flush; over a file the reader cannot read it refuses. `CultCache.FlushAsync(bool soft = false)`, `CultCache.FlushAllBackingStores(bool soft = false)`, `CultMeshNode.FlushAsync(soft)` and `CultNetHost.FlushAsync(soft)` pass the flag to the stores; `soft: true` was accepted and ignored before, so a caller that passed it now stops landing writes on an existing store. Semver: breaking for C# consumers with their own store kind. - A document payload with bytes after its first MessagePack value is refused on read and on write (`MessagePackSerializationException` naming the type and the count of bytes after the value, never the bytes); the tail is not walked, so a tail nested past the limit is refused as bytes after the value, not as depth. Before, the tail was ignored on read and dropped on the next write. A payload that ends early is still the deserializer's to refuse. A CDN manifest is a record payload and follows the same rule: the acceptance vectors `trailing_nil`, `trailing_reserved_code` and `trailing_truncated_array` in `contracts/cultmesh/content-acceptance-vectors.json` are now refusals. `CultMeshProviderSessionWire.DecodeCommandDocument` refuses a command record whose payload has bytes after its first value (`MessagePackSerializationException` with the count); before, the tail was ignored. Semver: breaking for C# consumers, a major bump. - A single-file store holding two records under one key is unreadable: the reader refuses it with `CultStoreUnreadableException` naming the key (keys compare by ordinal code units, no case folding or normalisation), so a cache, `ReadStoreSnapshot` and a flush refuse the file and leave it as it is. Before, the later record silently replaced the earlier. A directory store reads its manifest through the same reader and refuses a duplicate key the same way. Repair the duplicate with the runtime that wrote it before reading the store here. - A record whose payload carries a slot past the slots its reading type declares, at any depth (the record, a nested object, a list element, a union arm or a unit arm), is loaded read-only: it is served, and every staged write or removal of its key (`Upsert`, `UpsertVariant`, `Remove`, a batch, and `Flatten` of a variant) is refused with `CultSchemaConflictException` before anything is staged, and the store keeps the record byte for byte. Before, the extra slots were skipped on read and dropped on the next write. Declaring an id compatible consents only to depth-0 slots the schema comparison reports ignored under that id; any other extra slot keeps the record read-only whatever is declared, including under the reader's own id. A variant overriding a slot its type does not have is read-only too (it used to be shed on flush). Write such a record from the build that declares those slots. Missing trailing slots still default and stay writable. From 8df0c4b25a9519acf46253469dfdae002f6cc814 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 23:23:31 +0000 Subject: [PATCH 5/5] Soft flush: pin the host and node passthroughs with a held record (absent writes, readable skips, unreadable refuses); XML-doc the one meaning on the three FlushAsync(bool soft) overloads Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Caching/CultCache.cs | 5 ++ src/GameCult.Mesh/CultMesh.cs | 5 +- src/GameCult.Networking/CultNetLocal.cs | 5 +- .../NetworkingTests.cs | 64 +++++++++++++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 91dfa9ac..8b6ed01b 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2547,6 +2547,11 @@ namespace GameCult.Caching return true; }); + /// + /// Flushes every dirty backing store. With the flush has one meaning, decided by each store's + /// own read: a store with nothing at its path is written; a store that reads (empty included) is skipped and its + /// staged writes stay staged; a store that cannot be read refuses with . + /// public Task FlushAsync(bool soft = false) { FlushAllBackingStores(soft); diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 6ea48e0d..32b45e1f 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -428,7 +428,10 @@ namespace GameCult.Mesh } /// - /// Flushes the durable cache. + /// Flushes the durable cache. With the flush has one meaning, decided by the store's own + /// read and never by what the cache holds in memory: a store with nothing at its path is written; a store that reads + /// (empty included) is skipped and its staged writes stay staged; a store that cannot be read refuses with + /// and is left untouched. /// public Task FlushAsync(bool soft = false) { diff --git a/src/GameCult.Networking/CultNetLocal.cs b/src/GameCult.Networking/CultNetLocal.cs index e884cd8d..60937574 100644 --- a/src/GameCult.Networking/CultNetLocal.cs +++ b/src/GameCult.Networking/CultNetLocal.cs @@ -96,7 +96,10 @@ namespace GameCult.Networking } /// - /// Flushes the durable cache. + /// Flushes the durable cache. With the flush has one meaning, decided by the store's own + /// read and never by what the cache holds in memory: a store with nothing at its path is written; a store that reads + /// (empty included) is skipped and its staged writes stay staged; a store that cannot be read refuses with + /// and is left untouched. /// public Task FlushAsync(bool soft = false) { diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index b566dca3..1aee870d 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -7490,6 +7490,70 @@ namespace GameCult.Networking.Tests Assert.That(File.ReadAllBytes(cachePath), Is.Not.EqualTo(before), "a plain node flush after a skipped soft flush did not land the staged write"); } + // The cache HOLDS a record in every step: a passthrough that decides soft from the cache's memory instead of + // handing the store the flag fails the absent step (it skips and writes nothing) or the unreadable step (it + // never reaches the reader that refuses). + [Test] + public async Task CultNetLocal_Host_SoftFlush_With_A_Held_Record_Writes_Absent_Skips_Readable_Refuses_Unreadable() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultnet-host-soft-held-{Guid.NewGuid():N}.msgpack"); + CultNetHost? host = null; + try + { + host = await CultNetLocal.CreateHostAsync(filePath, new CultNetHostOptions { StartServer = false }); + await host.Cache.UpsertAsync(new PlayerData { PlayerId = Guid.NewGuid(), Email = "a@example.test", PasswordHash = "hash", Username = "First" }); + Assert.That(File.Exists(filePath), Is.False); + + await host.FlushAsync(soft: true); + Assert.That(File.Exists(filePath), Is.True, "a soft host flush over an absent store, with a record held, wrote nothing"); + var written = File.ReadAllBytes(filePath); + + await host.Cache.UpsertAsync(new PlayerData { PlayerId = Guid.NewGuid(), Email = "b@example.test", PasswordHash = "hash", Username = "Second" }); + await host.FlushAsync(soft: true); + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(written), "a soft host flush rewrote a readable store"); + + var junk = new byte[] { 0xC1, 0x01, 0x02, 0x03 }; + File.WriteAllBytes(filePath, junk); + Assert.That(async () => await host.FlushAsync(soft: true), Throws.InstanceOf()); + Assert.That(File.ReadAllBytes(filePath), Is.EqualTo(junk)); + } + finally + { + host?.Dispose(); + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + [Test] + public async Task CultMeshNode_SoftFlush_With_A_Held_Record_Writes_Absent_Skips_Readable_Refuses_Unreadable() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "cultmesh-soft-held", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + var cachePath = Path.Combine(rootPath, "world.ccmp"); + var documents = new CultNetDocumentRegistry() + .Register(CultNetDocumentBinding.ForDocument()); + using var node = await CultMesh.CreateNodeAsync(cachePath, new CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = new CultNetDatabaseOptions { RuntimeId = "soft-held-local", DocumentRegistry = documents } + }); + await node.Database.PutAsync(new CultRecordKey("note:one"), new MeshQuickstartNote { NoteId = "note:one", Body = "one" }); + Assert.That(File.Exists(cachePath), Is.False); + + await node.FlushAsync(soft: true); + Assert.That(File.Exists(cachePath), Is.True, "a soft node flush over an absent store, with a record held, wrote nothing"); + var written = File.ReadAllBytes(cachePath); + + await node.Database.PutAsync(new CultRecordKey("note:two"), new MeshQuickstartNote { NoteId = "note:two", Body = "two" }); + await node.FlushAsync(soft: true); + Assert.That(File.ReadAllBytes(cachePath), Is.EqualTo(written), "a soft node flush rewrote a readable store"); + + var junk = new byte[] { 0xC1, 0x01, 0x02, 0x03 }; + File.WriteAllBytes(cachePath, junk); + Assert.That(async () => await node.FlushAsync(soft: true), Throws.InstanceOf()); + Assert.That(File.ReadAllBytes(cachePath), Is.EqualTo(junk)); + } + [Test] public async Task CultNetLocal_CreateHostAsync_Wires_Durable_Cache_Without_Starting_Server() {