GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Make CultCache hydration subscription-shaped

Browse source
This commit is contained in:
Meta Vens 2026-07-20 01:48:56 +01:00
commit 085e177742
8 changed files with 554 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs
View file

@ -49,6 +49,12 @@ namespace GameCult.Caching.MessagePack
/// </summary>
public string? DirectoryStorePath { get; set; }
/// <summary>
/// Gets or sets the records to hydrate when opening a paged directory store.
/// The directory manifest is always read; rejected record payloads are never opened.
/// </summary>
public Func<CultPersistedRecordMetadata, bool>? DirectoryStoreHydrationFilter { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the directory backing store before opening.
/// </summary>
@ -82,7 +88,8 @@ namespace GameCult.Caching.MessagePack
{
var directoryStore = new DirectoryMessagePackBackingStore(filePath, options.DirectoryStorePath)
{
FlushOnDispose = options.StoreFlushOnDispose
FlushOnDispose = options.StoreFlushOnDispose,
HydrationFilter = options.DirectoryStoreHydrationFilter
};
options.ConfigureDirectoryStore?.Invoke(directoryStore);
cache.AddBackingStore(directoryStore);

26
src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
View file

@ -75,9 +75,18 @@ public static class CultDocumentMessagePackSerialization
/// </summary>
public static byte[] SerializeUntyped(object value, Type type)
{
return SerializeUntyped(value, type, CultDocumentRegistry.Shared);
}
/// <summary>
/// Serializes a value through an explicitly owned document registry.
/// </summary>
public static byte[] SerializeUntyped(object value, Type type, CultDocumentRegistry registry)
{
if (registry == null) throw new ArgumentNullException(nameof(registry));
if (value != null)
{
var descriptor = CultDocumentRegistry.Shared.GetRequired(type);
var descriptor = registry.GetRequired(type);
if (descriptor.GeneratedPayloadSerializer != null)
{
return descriptor.GeneratedPayloadSerializer(value);
@ -92,7 +101,16 @@ public static class CultDocumentMessagePackSerialization
/// </summary>
public static object DeserializeUntyped(Type type, byte[] payload)
{
var descriptor = CultDocumentRegistry.Shared.GetRequired(type);
return DeserializeUntyped(type, payload, CultDocumentRegistry.Shared);
}
/// <summary>
/// Deserializes a value through an explicitly owned document registry.
/// </summary>
public static object DeserializeUntyped(Type type, byte[] payload, CultDocumentRegistry registry)
{
if (registry == null) throw new ArgumentNullException(nameof(registry));
var descriptor = registry.GetRequired(type);
if (descriptor.GeneratedPayloadDeserializer != null)
{
return descriptor.GeneratedPayloadDeserializer(payload);
@ -444,7 +462,7 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
/// </summary>
protected override byte[] SerializePayload(object document)
{
return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType());
return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry);
}
/// <summary>
@ -452,6 +470,6 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
/// </summary>
protected override object DeserializePayload(Type documentType, byte[] payload)
{
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload);
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry);
}
}

349
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -16,10 +16,22 @@ namespace GameCult.Caching.MessagePack;
/// </summary>
public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
{
private const string IndexedFormatVersion = "cultcache.store.v2.directory-indexed";
private readonly FileInfo _manifestFile;
private readonly DirectoryInfo _recordDirectory;
private readonly ConcurrentDictionary<string, bool> _dirtyKeys = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, bool> _deletedKeys = new(StringComparer.Ordinal);
private readonly HashSet<string> _hydratedKeys = new(StringComparer.Ordinal);
private Dictionary<string, CultPersistedRecord> _durableIndex = new(StringComparer.Ordinal);
private Dictionary<string, CultPersistedRecord> _legacyInlineRecords = new(StringComparer.Ordinal);
private CultSchemaCatalogEntry[] _durableCatalog = Array.Empty<CultSchemaCatalogEntry>();
private bool _needsIndexUpgrade;
/// <summary>
/// Gets or sets the predicate that selects indexed record payloads for hydration.
/// A null predicate hydrates every record.
/// </summary>
public Func<CultPersistedRecordMetadata, bool>? HydrationFilter { get; set; }
/// <summary>
/// Creates a paged MessagePack backing store.
@ -52,46 +64,30 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
var manifest = ReadManifest();
_durableCatalog = manifest.SchemaCatalog;
var reports = new List<CultSchemaMigrationReport>();
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
foreach (var record in manifest.Records)
var indexed = string.Equals(manifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal);
if (indexed)
{
var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog);
reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog));
var stored = ToStoredDocument(record, catalog, CultDocumentMessagePackSerialization.DeserializeUntyped);
loaded[stored.Key.Value] = stored;
_needsIndexUpgrade = false;
_durableIndex = manifest.Records
.Where(record => !string.IsNullOrWhiteSpace(record.Key))
.GroupBy(record => record.Key, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal);
LoadIndexedRecords(manifest, loaded, reports);
}
if (_recordDirectory.Exists)
else
{
var recordFiles = _recordDirectory
.EnumerateFiles("*.msgpack")
.OrderBy(file => file.Name, StringComparer.Ordinal)
.ToArray();
var recordReports = new CultSchemaMigrationReport[recordFiles.Length];
var storedRecords = new CultStoredDocument[recordFiles.Length];
Parallel.For(
0,
recordFiles.Length,
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 8) },
index =>
{
var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(
ReadAllBytesShared(recordFiles[index].FullName));
var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog);
recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog);
storedRecords[index] = ToStoredDocument(
record,
catalog,
CultDocumentMessagePackSerialization.DeserializeUntyped);
});
for (var index = 0; index < storedRecords.Length; index++)
{
reports.Add(recordReports[index]);
var stored = storedRecords[index];
loaded[stored.Key.Value] = stored;
}
// V1 manifests have no page index. Hydrate once to discover the durable keys;
// the next flush replaces this compatibility path with the indexed manifest.
_needsIndexUpgrade = true;
var legacyRecords = LoadLegacyRecords(manifest, loaded, reports);
_durableIndex = legacyRecords
.Where(record => !string.IsNullOrWhiteSpace(record.Key))
.GroupBy(record => record.Key, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => ToIndexRecord(group.Last()), StringComparer.Ordinal);
}
foreach (var pair in loaded)
@ -113,15 +109,47 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
EntryUpdated.OnNext(pair.Value);
}
foreach (var removedKey in Entries.Keys.Where(key => !loaded.ContainsKey(key)).ToArray())
foreach (var removedKey in _hydratedKeys.Where(key => !loaded.ContainsKey(key)).ToArray())
{
if (_dirtyKeys.ContainsKey(removedKey) || _deletedKeys.ContainsKey(removedKey))
continue;
if (Entries.TryRemove(removedKey, out var removed))
EntryDeleted.OnNext(removed);
_hydratedKeys.Remove(removedKey);
}
foreach (var key in loaded.Keys)
_hydratedKeys.Add(key);
SetLastSchemaMigrationReports(reports);
IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty;
}
/// <inheritdoc />
public override void PullSelected(Func<CultPersistedRecordMetadata, bool> selector)
{
if (selector == null) throw new ArgumentNullException(nameof(selector));
var manifest = ReadManifest();
if (!string.Equals(manifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal))
{
PullAll();
return;
}
_durableCatalog = manifest.SchemaCatalog;
_durableIndex = manifest.Records
.Where(record => !string.IsNullOrWhiteSpace(record.Key))
.GroupBy(record => record.Key, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal);
var selected = manifest.Records
.Where(record => selector(ToMetadata(record)))
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray();
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
var reports = new List<CultSchemaMigrationReport>();
LoadRecordPages(selected, manifest.SchemaCatalog, loaded, reports);
PublishSelected(loaded, selected.Select(record => record.Key).ToHashSet(StringComparer.Ordinal));
SetLastSchemaMigrationReports(reports);
IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty;
}
@ -150,18 +178,46 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Directory.CreateDirectory(_manifestFile.DirectoryName!);
Directory.CreateDirectory(_recordDirectory.FullName);
var targetCatalog = BuildTargetCatalog();
var durableCatalog = ReadManifest().SchemaCatalog;
var precommitCatalog = durableCatalog
var currentManifest = ReadManifest();
var currentIndex = string.Equals(currentManifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal)
? currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal)
: new Dictionary<string, CultPersistedRecord>(_durableIndex, StringComparer.Ordinal);
var precommitIndex = currentIndex.Values
.Select(ToIndexRecord)
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray();
foreach (var key in _deletedKeys.Keys)
currentIndex.Remove(key);
foreach (var key in _dirtyKeys.Keys)
{
if (Entries.TryGetValue(key, out var stored))
currentIndex[key] = ToIndexRecord(stored);
}
var catalogCandidates = currentManifest.SchemaCatalog
.Concat(_durableCatalog)
.Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()))
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.Last())
.ToArray();
var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal);
var targetCatalog = catalogCandidates
.Where(entry => usedSchemaIds.Contains(entry.SchemaId))
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ToArray();
var precommitCatalog = currentManifest.SchemaCatalog
.Concat(targetCatalog)
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.Last())
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ToArray();
WriteManifest(precommitCatalog);
WriteManifest(precommitCatalog, precommitIndex);
foreach (var key in _dirtyKeys.Keys.OrderBy(value => value, StringComparer.Ordinal))
var keysToWrite = (_needsIndexUpgrade ? Entries.Keys : _dirtyKeys.Keys)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
foreach (var key in keysToWrite)
{
if (!Entries.TryGetValue(key, out var stored))
{
@ -169,10 +225,22 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
var record = ToPersistedRecord(stored, document =>
CultDocumentMessagePackSerialization.SerializeUntyped(document, stored.Descriptor.DocumentType));
CultDocumentMessagePackSerialization.SerializeUntyped(
document,
stored.Descriptor.DocumentType,
Registry));
WriteFileAtomically(RecordPath(key), CultDocumentMessagePackSerialization.SerializePersistedRecord(record));
}
foreach (var pair in _legacyInlineRecords.OrderBy(pair => pair.Key, StringComparer.Ordinal))
{
if (_deletedKeys.ContainsKey(pair.Key) || _dirtyKeys.ContainsKey(pair.Key))
continue;
WriteFileAtomically(
RecordPath(pair.Key),
CultDocumentMessagePackSerialization.SerializePersistedRecord(pair.Value));
}
foreach (var key in _deletedKeys.Keys.OrderBy(value => value, StringComparer.Ordinal))
{
var path = RecordPath(key);
@ -182,38 +250,199 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
}
if (!CatalogsEqual(precommitCatalog, targetCatalog))
{
WriteManifest(targetCatalog);
}
WriteManifest(targetCatalog, currentIndex.Values
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray());
_durableCatalog = targetCatalog;
_durableIndex = currentIndex;
_legacyInlineRecords.Clear();
_needsIndexUpgrade = false;
_dirtyKeys.Clear();
_deletedKeys.Clear();
MarkFlushSucceeded();
}
private CultSchemaCatalogEntry[] BuildTargetCatalog() => Entries.Values
.Select(entry => entry.Descriptor.ToCatalogEntry())
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ToArray();
private static bool CatalogsEqual(CultSchemaCatalogEntry[] left, CultSchemaCatalogEntry[] right) =>
CultDocumentMessagePackSerialization.SerializeSchemaCatalog(left)
.SequenceEqual(CultDocumentMessagePackSerialization.SerializeSchemaCatalog(right));
private void WriteManifest(CultSchemaCatalogEntry[] catalog)
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index)
{
var manifest = new CultPersistedStoreSnapshot
{
FormatVersion = "cultcache.store.v1.directory",
FormatVersion = IndexedFormatVersion,
SchemaCatalog = catalog,
Records = Array.Empty<CultPersistedRecord>()
Records = index
};
WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest));
}
private void LoadIndexedRecords(
CultPersistedStoreSnapshot manifest,
Dictionary<string, CultStoredDocument> loaded,
List<CultSchemaMigrationReport> reports)
{
var selected = manifest.Records
.Where(ShouldHydrate)
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray();
LoadRecordPages(selected, manifest.SchemaCatalog, loaded, reports);
}
private CultPersistedRecord[] LoadLegacyRecords(
CultPersistedStoreSnapshot manifest,
Dictionary<string, CultStoredDocument> loaded,
List<CultSchemaMigrationReport> reports)
{
_legacyInlineRecords = manifest.Records
.Where(record => !string.IsNullOrWhiteSpace(record.Key))
.GroupBy(record => record.Key, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal);
foreach (var record in manifest.Records.Where(ShouldHydrate))
{
var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog);
reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog));
var stored = ToStoredDocument(
record,
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry));
loaded[stored.Key.Value] = stored;
}
if (!_recordDirectory.Exists)
return manifest.Records;
var pages = _recordDirectory.EnumerateFiles("*.msgpack")
.OrderBy(file => file.Name, StringComparer.Ordinal)
.Select(file => CultDocumentMessagePackSerialization.DeserializePersistedRecord(ReadAllBytesShared(file.FullName)))
.ToArray();
LoadPersistedRecords(pages.Where(ShouldHydrate).ToArray(), manifest.SchemaCatalog, loaded, reports);
return manifest.Records.Concat(pages).ToArray();
}
private void LoadRecordPages(
CultPersistedRecord[] records,
IReadOnlyCollection<CultSchemaCatalogEntry> catalogEntries,
Dictionary<string, CultStoredDocument> loaded,
List<CultSchemaMigrationReport> reports)
{
var recordReports = new CultSchemaMigrationReport[records.Length];
var storedRecords = new CultStoredDocument?[records.Length];
Parallel.For(
0,
records.Length,
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 8) },
index =>
{
var metadata = records[index];
var path = RecordPath(metadata.Key);
if (!File.Exists(path))
return;
var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(ReadAllBytesShared(path));
if (!string.Equals(record.Key, metadata.Key, StringComparison.Ordinal))
throw new InvalidDataException($"Record page '{path}' contains key '{record.Key}', expected '{metadata.Key}'.");
var catalog = ResolveLegacyUncataloguedRecordCatalog(record, catalogEntries);
recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog);
storedRecords[index] = ToStoredDocument(
record,
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry));
});
for (var index = 0; index < storedRecords.Length; index++)
{
var stored = storedRecords[index];
if (stored == null)
continue;
reports.Add(recordReports[index]);
loaded[stored.Key.Value] = stored;
}
}
private void LoadPersistedRecords(
CultPersistedRecord[] records,
IReadOnlyCollection<CultSchemaCatalogEntry> catalogEntries,
Dictionary<string, CultStoredDocument> loaded,
List<CultSchemaMigrationReport> reports)
{
var recordReports = new CultSchemaMigrationReport[records.Length];
var storedRecords = new CultStoredDocument[records.Length];
Parallel.For(
0,
records.Length,
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 8) },
index =>
{
var record = records[index];
var catalog = ResolveLegacyUncataloguedRecordCatalog(record, catalogEntries);
recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog);
storedRecords[index] = ToStoredDocument(
record,
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry));
});
for (var index = 0; index < storedRecords.Length; index++)
{
reports.Add(recordReports[index]);
var stored = storedRecords[index];
loaded[stored.Key.Value] = stored;
}
}
private bool ShouldHydrate(CultPersistedRecord record) =>
HydrationFilter?.Invoke(ToMetadata(record)) ?? true;
private static CultPersistedRecordMetadata ToMetadata(CultPersistedRecord record) =>
new(record.Key, record.SchemaId, record.StoredAt);
private void PublishSelected(
IReadOnlyDictionary<string, CultStoredDocument> loaded,
HashSet<string> selectedKeys)
{
foreach (var pair in loaded)
{
if (_dirtyKeys.ContainsKey(pair.Key) || _deletedKeys.ContainsKey(pair.Key))
continue;
if (Entries.TryGetValue(pair.Key, out var existing) &&
string.Equals(existing.StoredAt, pair.Value.StoredAt, StringComparison.Ordinal) &&
string.Equals(existing.Descriptor.SchemaId, pair.Value.Descriptor.SchemaId, StringComparison.Ordinal))
continue;
Entries[pair.Key] = pair.Value;
if (existing == null)
EntryAdded.OnNext(pair.Value);
else
EntryUpdated.OnNext(pair.Value);
}
foreach (var missingKey in selectedKeys.Where(key => !loaded.ContainsKey(key)).ToArray())
{
if (_dirtyKeys.ContainsKey(missingKey) || _deletedKeys.ContainsKey(missingKey))
continue;
if (Entries.TryRemove(missingKey, out var removed))
EntryDeleted.OnNext(removed);
_hydratedKeys.Remove(missingKey);
}
foreach (var key in loaded.Keys)
_hydratedKeys.Add(key);
}
private static CultPersistedRecord ToIndexRecord(CultStoredDocument stored) => new()
{
Key = stored.Key.Value,
SchemaId = stored.Descriptor.SchemaId,
StoredAt = stored.StoredAt,
Payload = Array.Empty<byte>()
};
private static CultPersistedRecord ToIndexRecord(CultPersistedRecord record) => new()
{
Key = record.Key,
SchemaId = record.SchemaId,
StoredAt = record.StoredAt,
Payload = Array.Empty<byte>()
};
private CultPersistedStoreSnapshot ReadManifest()
{
if (File.Exists(_manifestFile.FullName))

67
src/GameCult.Caching/CultCache.cs
View file

@ -420,6 +420,28 @@ namespace GameCult.Caching
Refresh();
}
private CultDocumentRegistry(bool discoverLoadedDocuments)
{
if (discoverLoadedDocuments)
Refresh();
}
/// <summary>
/// Creates a registry containing exactly the requested document types without scanning loaded assemblies.
/// </summary>
public static CultDocumentRegistry ForTypes(IEnumerable<Type> documentTypes)
{
if (documentTypes == null) throw new ArgumentNullException(nameof(documentTypes));
var registry = new CultDocumentRegistry(discoverLoadedDocuments: false);
foreach (var type in documentTypes.Distinct())
{
registry.GetRequired(type ?? throw new ArgumentException(
"Document type collections cannot contain null entries.",
nameof(documentTypes)));
}
return registry;
}
/// <summary>
/// Gets all known document descriptors.
/// </summary>
@ -1392,6 +1414,22 @@ namespace GameCult.Caching
await Task.CompletedTask;
}
/// <summary>
/// Pulls persisted records selected by durable metadata from every attached backing store.
/// Stores without indexed selection support preserve correctness by performing a full pull.
/// </summary>
public async Task PullBackingStoreRecordsAsync(Func<CultPersistedRecordMetadata, bool> selector)
{
if (selector == null) throw new ArgumentNullException(nameof(selector));
foreach (var store in _backingStores)
{
store.PullSelected(selector);
}
_hasUnflushedMutations = _backingStores.Any(store => store.IsDirty);
await Task.CompletedTask;
}
/// <summary>
/// Flushes all attached backing stores.
/// </summary>
@ -1892,6 +1930,27 @@ namespace GameCult.Caching
}
}
/// <summary>
/// Durable metadata available before a backing store hydrates a record payload.
/// </summary>
public sealed class CultPersistedRecordMetadata
{
/// <summary>Creates durable record metadata.</summary>
public CultPersistedRecordMetadata(string key, string schemaId, string storedAt)
{
Key = key;
SchemaId = schemaId;
StoredAt = storedAt;
}
/// <summary>Gets the durable record key.</summary>
public string Key { get; }
/// <summary>Gets the durable schema identity.</summary>
public string SchemaId { get; }
/// <summary>Gets the record commit timestamp.</summary>
public string StoredAt { get; }
}
/// <summary>
/// Base class for CultCache persistence adapters.
/// </summary>
@ -1970,6 +2029,14 @@ namespace GameCult.Caching
/// </summary>
public abstract void PullAll();
/// <summary>
/// Pulls records selected by durable metadata. Non-indexed stores fall back to a full pull.
/// </summary>
public virtual void PullSelected(Func<CultPersistedRecordMetadata, bool> selector)
{
if (selector == null) throw new ArgumentNullException(nameof(selector));
PullAll();
}
/// <summary>
/// Pushes one stored document into the backing store.
/// </summary>
public abstract void Push(CultStoredDocument entry);

8
src/GameCult.Mesh/CultMeshDocumentRegistries.cs
View file

@ -26,13 +26,7 @@ namespace GameCult.Mesh
throw new ArgumentNullException(nameof(documentTypes));
}
var registry = new CultDocumentRegistry();
foreach (var documentType in DistinctDocumentTypes(documentTypes))
{
registry.GetRequired(documentType);
}
return registry;
return CultDocumentRegistry.ForTypes(DistinctDocumentTypes(documentTypes));
}
/// <summary>

28
src/GameCult.Mesh/docs/reliability-migration-plan.md
View file

@ -47,6 +47,34 @@ usable while endpoints fail or move.
## Target Authority Map
### Local persistence hydration
**Owner:** the attached CultCache backing store owns durable record discovery;
the consuming service owns its working-set selector.
**Inputs:** an indexed manifest containing record key, schema identity, and
commit time; an explicit metadata predicate supplied at open or at a later
subscription boundary.
**Outputs:** only selected typed documents enter the live cache. Record payload
pages outside the working set remain durable and unopened. Indexed stores may
hydrate an additional selection without replaying or evicting the existing hot
set; non-indexed stores retain the correct full-pull fallback.
**Derived state:** hydrated keys, dirty keys, and migration reports are local
backing-store state. The manifest index remains the durable catalog even when a
consumer has only a narrow working set in memory.
**Forbidden writers:** service boot code must not scan page directories or
deserialize broad snapshots to discover keys. A filtered flush must not delete
unsubscribed records, and implicit global assembly discovery must not expand an
explicit service registry.
**Cut line:** directory manifests now carry metadata-only record indexes;
explicit registries contain exactly the requested document types; selective
pulls use the same indexed authority instead of reopening the store through a
second compatibility path.
### Discovery control plane
**Owner:** `CultMeshDiscoveryService`.

127
tests/GameCult.Caching.Tests/BackingStoreTests.cs
View file

@ -321,6 +321,133 @@ namespace GameCult.Caching.Tests
}
}
[Test]
public async Task DirectoryMessagePackBackingStore_IndexedFilter_DoesNotOpenOrDeleteColdPayloads()
{
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.cc");
var recordsPath = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath);
try
{
CultRecordHandle<NamedTestEntry> hot;
CultRecordHandle<NamedTestEntry> cold;
using (var seed = await CultCacheMessagePack.OpenAsync(
filePath,
new CultCacheOpenOptions { UseDirectoryStore = true }))
{
hot = await seed.UpsertAsync(new NamedTestEntry { Name = "hot", Value = "hydrate-me" });
cold = await seed.UpsertAsync(new NamedTestEntry
{
Name = "cold",
Value = new string('c', 1024 * 1024)
});
await seed.FlushAsync();
}
var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(filePath));
Assert.That(manifest.FormatVersion, Is.EqualTo("cultcache.store.v2.directory-indexed"));
Assert.That(manifest.Records, Has.Length.EqualTo(2));
Assert.That(manifest.Records.All(record => record.Payload.Length == 0), Is.True);
var coldPath = Directory.GetFiles(recordsPath, "*.msgpack")
.Single(path => string.Equals(
CultDocumentMessagePackSerialization.DeserializePersistedRecord(File.ReadAllBytes(path)).Key,
cold.Key.Value,
StringComparison.Ordinal));
using (new FileStream(coldPath, FileMode.Open, FileAccess.Read, FileShare.None))
using (var selected = await CultCacheMessagePack.OpenAsync(
filePath,
new CultCacheOpenOptions
{
UseDirectoryStore = true,
DirectoryStoreHydrationFilter = metadata =>
string.Equals(metadata.Key, hot.Key.Value, StringComparison.Ordinal)
}))
{
Assert.That(selected.Get<NamedTestEntry>(hot.Key)?.Value, Is.EqualTo("hydrate-me"));
Assert.That(selected.Get<NamedTestEntry>(cold.Key), Is.Null);
await selected.UpsertAsync(new NamedTestEntry { Name = "new", Value = "persist-with-cold-page-locked" });
await selected.FlushAsync();
}
using var reopened = await CultCacheMessagePack.OpenAsync(
filePath,
new CultCacheOpenOptions { UseDirectoryStore = true });
Assert.That(reopened.Get<NamedTestEntry>(cold.Key)?.Value.Length, Is.EqualTo(1024 * 1024));
Assert.That(reopened.GetAll<NamedTestEntry>().Count(), Is.EqualTo(3));
var finalManifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(filePath));
Assert.That(finalManifest.Records, Has.Length.EqualTo(3));
Assert.That(finalManifest.Records.Any(record => record.Key == cold.Key.Value), Is.True);
Assert.That(finalManifest.Records.All(record => record.Payload.Length == 0), Is.True);
}
finally
{
if (File.Exists(filePath))
File.Delete(filePath);
if (Directory.Exists(recordsPath))
Directory.Delete(recordsPath, recursive: true);
}
}
[Test]
public async Task DirectoryMessagePackBackingStore_PullSelected_HydratesMatchingColdPagesOnly()
{
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.cc");
var recordsPath = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath);
try
{
CultRecordHandle<NamedTestEntry> hot;
CultRecordHandle<NamedTestEntry> requested;
CultRecordHandle<NamedTestEntry> unrelated;
using (var seed = await CultCacheMessagePack.OpenAsync(
filePath,
new CultCacheOpenOptions { UseDirectoryStore = true }))
{
hot = await seed.UpsertAsync(new NamedTestEntry { Name = "hot", Value = "already-loaded" });
requested = await seed.UpsertAsync(new NamedTestEntry { Name = "requested", Value = "load-later" });
unrelated = await seed.UpsertAsync(new NamedTestEntry
{
Name = "unrelated",
Value = new string('u', 1024 * 1024)
});
await seed.FlushAsync();
}
string PageFor(CultRecordHandle<NamedTestEntry> handle) =>
Directory.GetFiles(recordsPath, "*.msgpack").Single(path => string.Equals(
CultDocumentMessagePackSerialization.DeserializePersistedRecord(File.ReadAllBytes(path)).Key,
handle.Key.Value,
StringComparison.Ordinal));
using var selected = await CultCacheMessagePack.OpenAsync(
filePath,
new CultCacheOpenOptions
{
UseDirectoryStore = true,
DirectoryStoreHydrationFilter = metadata => metadata.Key == hot.Key.Value
});
Assert.That(selected.Get<NamedTestEntry>(requested.Key), Is.Null);
using (new FileStream(PageFor(unrelated), FileMode.Open, FileAccess.Read, FileShare.None))
{
await selected.PullBackingStoreRecordsAsync(metadata => metadata.Key == requested.Key.Value);
Assert.That(selected.Get<NamedTestEntry>(requested.Key)?.Value, Is.EqualTo("load-later"));
Assert.That(selected.Get<NamedTestEntry>(hot.Key)?.Value, Is.EqualTo("already-loaded"));
Assert.That(selected.Get<NamedTestEntry>(unrelated.Key), Is.Null);
}
}
finally
{
if (File.Exists(filePath))
File.Delete(filePath);
if (Directory.Exists(recordsPath))
Directory.Delete(recordsPath, recursive: true);
}
}
[Test]
public async Task DirectoryMessagePackBackingStore_Loads_Record_When_Manifest_Misses_Catalog_Entry()
{

12
tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs
View file

@ -10,6 +10,18 @@ namespace GameCult.Caching.Tests;
[TestFixture]
public sealed class CultDocumentRegistryTests
{
[Test]
public void ForTypesContainsOnlyExplicitDocuments()
{
var requested = CreateDocumentType("explicit", "tests.registry.explicit", "v1");
var unrelated = CreateDocumentType("unrelated", "tests.registry.unrelated", "v1");
var registry = CultDocumentRegistry.ForTypes(new[] { requested, requested });
Assert.That(registry.AllDescriptors.Select(value => value.DocumentType), Is.EqualTo(new[] { requested }));
Assert.That(registry.AllDescriptors.Any(value => value.DocumentType == unrelated), Is.False);
}
[Test]
public void IdenticalTypeRegistrationIsIdempotent()
{