From a32271d7c22f99b957279d595fcf23f8de710945 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 18 Jul 2026 18:35:39 +0100 Subject: [PATCH] Make CultCache multiprocess pulls incremental --- .../DirectoryMessagePackBackingStore.cs | 49 +++++++--- src/GameCult.Caching/CultCache.cs | 41 ++++++++- .../BackingStoreTests.cs | 92 +++++++++++++++++++ 3 files changed, 168 insertions(+), 14 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 7104705d..1597be2a 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -42,6 +42,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore /// public override void PullAll() { + _recordDirectory.Refresh(); if (!File.Exists(_manifestFile.FullName) && !_recordDirectory.Exists) { SetLastSchemaMigrationReports(Array.Empty()); @@ -51,37 +52,53 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var manifest = ReadManifest(); var reports = new List(); + var loaded = new Dictionary(StringComparer.Ordinal); foreach (var record in manifest.Records) { var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog); reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog)); var stored = ToStoredDocument(record, catalog, CultDocumentMessagePackSerialization.DeserializeUntyped); - Entries[stored.Key.Value] = stored; - _dirtyKeys[stored.Key.Value] = true; - EntryAdded.OnNext(stored); + loaded[stored.Key.Value] = stored; } if (_recordDirectory.Exists) { foreach (var recordFile in _recordDirectory.EnumerateFiles("*.msgpack").OrderBy(file => file.Name, StringComparer.Ordinal)) { - var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(File.ReadAllBytes(recordFile.FullName)); + var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(ReadAllBytesShared(recordFile.FullName)); var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog); reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog)); var stored = ToStoredDocument(record, catalog, CultDocumentMessagePackSerialization.DeserializeUntyped); - Entries[stored.Key.Value] = stored; - EntryAdded.OnNext(stored); + loaded[stored.Key.Value] = stored; } } - SetLastSchemaMigrationReports(reports); - if (manifest.Records.Length == 0) + foreach (var pair in loaded) { - _dirtyKeys.Clear(); + if (Entries.TryGetValue(pair.Key, out var existing) && + string.Equals(existing.StoredAt, pair.Value.StoredAt, StringComparison.Ordinal) && + string.Equals(existing.Descriptor.SchemaId, pair.Value.Descriptor.SchemaId, StringComparison.Ordinal)) + { + continue; + } + + Entries[pair.Key] = pair.Value; + if (existing == null) + EntryAdded.OnNext(pair.Value); + else + EntryUpdated.OnNext(pair.Value); } + foreach (var removedKey in Entries.Keys.Where(key => !loaded.ContainsKey(key)).ToArray()) + { + if (Entries.TryRemove(removedKey, out var removed)) + EntryDeleted.OnNext(removed); + } + + SetLastSchemaMigrationReports(reports); + _dirtyKeys.Clear(); _deletedKeys.Clear(); - IsDirty = _dirtyKeys.Count > 0; + IsDirty = false; } /// @@ -176,7 +193,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { if (File.Exists(_manifestFile.FullName)) { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_manifestFile.FullName)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(ReadAllBytesShared(_manifestFile.FullName)); if (snapshot.SchemaCatalog.Length > 0 || snapshot.Records.Length == 0) { return snapshot; @@ -185,7 +202,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore if (File.Exists(_manifestFile.FullName)) { - return CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_manifestFile.FullName)); + return CultDocumentMessagePackSerialization.DeserializeSnapshot(ReadAllBytesShared(_manifestFile.FullName)); } return new CultPersistedStoreSnapshot @@ -317,6 +334,14 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore } } + private static byte[] ReadAllBytesShared(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } + private static void ReplaceExistingFile(string sourcePath, string destinationPath) { for (var attempt = 0; attempt < 3; attempt++) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index e57cd6df..ba9bdbdf 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2084,6 +2084,7 @@ namespace GameCult.Caching /// public override void PullAll() { + FileInfo.Refresh(); if (!FileInfo.Exists) { SetLastSchemaMigrationReports(Array.Empty()); @@ -2091,14 +2092,42 @@ namespace GameCult.Caching return; } - var snapshot = DeserializeSnapshot(File.ReadAllBytes(FileInfo.FullName)); + CultPersistedStoreSnapshot snapshot; + try + { + snapshot = DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); + } + catch (FileNotFoundException) + { + FileInfo.Refresh(); + if (!FileInfo.Exists) + return; + snapshot = DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); + } var reports = new List(snapshot.Records.Length); + var persistedKeys = new HashSet(StringComparer.Ordinal); foreach (var record in snapshot.Records) { + persistedKeys.Add(record.Key); reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog)); var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload); + if (Entries.TryGetValue(stored.Key.Value, out var existing) && + string.Equals(existing.StoredAt, stored.StoredAt, StringComparison.Ordinal) && + string.Equals(existing.Descriptor.SchemaId, stored.Descriptor.SchemaId, StringComparison.Ordinal)) + { + continue; + } Entries[stored.Key.Value] = stored; - EntryAdded.OnNext(stored); + if (existing == null) + EntryAdded.OnNext(stored); + else + EntryUpdated.OnNext(stored); + } + + foreach (var removedKey in Entries.Keys.Where(key => !persistedKeys.Contains(key)).ToArray()) + { + if (Entries.TryRemove(removedKey, out var removed)) + EntryDeleted.OnNext(removed); } SetLastSchemaMigrationReports(reports); @@ -2183,5 +2212,13 @@ namespace GameCult.Caching } } } + + private static byte[] ReadAllBytesShared(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } } } diff --git a/tests/GameCult.Caching.Tests/BackingStoreTests.cs b/tests/GameCult.Caching.Tests/BackingStoreTests.cs index 18cf8882..328b778e 100644 --- a/tests/GameCult.Caching.Tests/BackingStoreTests.cs +++ b/tests/GameCult.Caching.Tests/BackingStoreTests.cs @@ -54,6 +54,59 @@ namespace GameCult.Caching.Tests } } + [Test] + public async Task SingleFileMessagePackBackingStore_PullAll_SeesFileCreatedAfterReaderOpened() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack"); + + try + { + var readCache = new CultCache(); + readCache.AddBackingStore(new SingleFileMessagePackBackingStore(filePath)); + await readCache.PullAllBackingStoresAsync(); + + var observedChanges = 0; + using var subscription = readCache.Watch() + .Subscribe(_ => observedChanges++); + + var writeStore = new SingleFileMessagePackBackingStore(filePath); + var writeCache = new CultCache(); + writeCache.AddBackingStore(writeStore); + var handle = await writeCache.AddAsync(new NamedTestEntry + { + Name = "external-writer", + Value = "visible-after-refresh" + }); + writeStore.PushAll(); + + await readCache.PullAllBackingStoresAsync(); + + Assert.That(readCache.Get(handle.Key)?.Value, Is.EqualTo("visible-after-refresh")); + Assert.That(observedChanges, Is.EqualTo(1)); + + await readCache.PullAllBackingStoresAsync(); + Assert.That(observedChanges, Is.EqualTo(1), "an unchanged snapshot must not replay its documents"); + + await writeCache.UpsertAsync(new NamedTestEntry + { + Name = "external-writer", + Value = "updated-once" + }, handle); + writeStore.PushAll(); + await readCache.PullAllBackingStoresAsync(); + + Assert.That(readCache.Get(handle.Key)?.Value, Is.EqualTo("updated-once")); + Assert.That(observedChanges, Is.EqualTo(2)); + } + finally + { + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + } + } + [Test] public async Task CultCache_DirtyState_Tracks_Mutations_And_ExplicitFlush() { @@ -333,6 +386,45 @@ namespace GameCult.Caching.Tests } } + [Test] + public async Task DirectoryMessagePackBackingStore_PullsOnlyExternalRecordDeltas() + { + var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.cc"); + var recordsPath = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath); + + try + { + using var reader = await CultCacheMessagePack.OpenAsync( + filePath, + new CultCacheOpenOptions { UseDirectoryStore = true }); + var observedChanges = 0; + using var subscription = reader.Watch().Subscribe(_ => observedChanges++); + + using var writer = await CultCacheMessagePack.OpenAsync( + filePath, + new CultCacheOpenOptions { UseDirectoryStore = true }); + var handle = await writer.UpsertAsync(new NamedTestEntry + { + Name = "external-command", + Value = "once" + }); + await writer.FlushAsync(); + + await reader.PullAllBackingStoresAsync(); + await reader.PullAllBackingStoresAsync(); + + Assert.That(reader.Get(handle.Key)?.Value, Is.EqualTo("once")); + Assert.That(observedChanges, Is.EqualTo(1), "unchanged paged records must not replay"); + } + finally + { + if (File.Exists(filePath)) + File.Delete(filePath); + if (Directory.Exists(recordsPath)) + Directory.Delete(recordsPath, recursive: true); + } + } + [Test] public async Task DirectoryMessagePackBackingStore_Remains_Dirty_When_Deletion_Fails() {