Hand-write CultCache MessagePack store format
This commit is contained in:
Meta Vens
parent
c26870b803
commit
ef1f4c16ee
2 changed files with 304 additions and 6 deletions
|
|
@ -1,4 +1,5 @@
|
|||
using System;
|
||||
using System.Buffers;
|
||||
using GameCult.Caching;
|
||||
using MessagePack;
|
||||
using MessagePack.Formatters;
|
||||
|
|
@ -26,6 +27,10 @@ public sealed class CultDocumentResolver : IFormatterResolver
|
|||
|
||||
public static class CultDocumentMessagePackSerialization
|
||||
{
|
||||
private const int PersistedRecordFieldCount = 4;
|
||||
private const int SchemaCatalogEntryFieldCount = 6;
|
||||
private const int StoreSnapshotFieldCount = 3;
|
||||
|
||||
public static readonly MessagePackSerializerOptions Options =
|
||||
MessagePackSerializerOptions.Standard
|
||||
.WithResolver(CompositeResolver.Create(
|
||||
|
|
@ -67,6 +72,214 @@ public static class CultDocumentMessagePackSerialization
|
|||
|
||||
return MessagePackSerializer.Deserialize(type, payload, Options);
|
||||
}
|
||||
|
||||
public static byte[] SerializePersistedRecord(CultPersistedRecord record)
|
||||
{
|
||||
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
|
||||
var writer = new MessagePackWriter(buffer);
|
||||
WritePersistedRecord(ref writer, record);
|
||||
writer.Flush();
|
||||
return buffer.WrittenSpan.ToArray();
|
||||
}
|
||||
|
||||
public static CultPersistedRecord DeserializePersistedRecord(byte[] payload)
|
||||
{
|
||||
var reader = new MessagePackReader(payload);
|
||||
return ReadPersistedRecord(ref reader);
|
||||
}
|
||||
|
||||
public static byte[] SerializeSchemaCatalog(CultSchemaCatalogEntry[] catalog)
|
||||
{
|
||||
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
|
||||
var writer = new MessagePackWriter(buffer);
|
||||
writer.WriteArrayHeader(catalog.Length);
|
||||
foreach (var entry in catalog)
|
||||
{
|
||||
WriteSchemaCatalogEntry(ref writer, entry);
|
||||
}
|
||||
|
||||
writer.Flush();
|
||||
return buffer.WrittenSpan.ToArray();
|
||||
}
|
||||
|
||||
public static CultSchemaCatalogEntry[] DeserializeSchemaCatalog(byte[] payload)
|
||||
{
|
||||
var reader = new MessagePackReader(payload);
|
||||
var count = reader.ReadArrayHeader();
|
||||
var catalog = new CultSchemaCatalogEntry[count];
|
||||
for (var index = 0; index < count; index++)
|
||||
{
|
||||
catalog[index] = ReadSchemaCatalogEntry(ref reader);
|
||||
}
|
||||
|
||||
return catalog;
|
||||
}
|
||||
|
||||
public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
|
||||
{
|
||||
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
|
||||
var writer = new MessagePackWriter(buffer);
|
||||
writer.WriteArrayHeader(StoreSnapshotFieldCount);
|
||||
writer.Write(snapshot.FormatVersion);
|
||||
writer.WriteArrayHeader(snapshot.SchemaCatalog.Length);
|
||||
foreach (var entry in snapshot.SchemaCatalog)
|
||||
{
|
||||
WriteSchemaCatalogEntry(ref writer, entry);
|
||||
}
|
||||
|
||||
writer.WriteArrayHeader(snapshot.Records.Length);
|
||||
foreach (var record in snapshot.Records)
|
||||
{
|
||||
WritePersistedRecord(ref writer, record);
|
||||
}
|
||||
|
||||
writer.Flush();
|
||||
return buffer.WrittenSpan.ToArray();
|
||||
}
|
||||
|
||||
public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload)
|
||||
{
|
||||
var reader = new MessagePackReader(payload);
|
||||
var fieldCount = reader.ReadArrayHeader();
|
||||
var snapshot = new CultPersistedStoreSnapshot();
|
||||
|
||||
if (fieldCount > 0)
|
||||
{
|
||||
snapshot.FormatVersion = reader.ReadString() ?? "cultcache.store.v1";
|
||||
}
|
||||
|
||||
if (fieldCount > 1)
|
||||
{
|
||||
var catalogCount = reader.ReadArrayHeader();
|
||||
snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
|
||||
for (var index = 0; index < catalogCount; index++)
|
||||
{
|
||||
snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader);
|
||||
}
|
||||
}
|
||||
|
||||
if (fieldCount > 2)
|
||||
{
|
||||
var recordCount = reader.ReadArrayHeader();
|
||||
snapshot.Records = new CultPersistedRecord[recordCount];
|
||||
for (var index = 0; index < recordCount; index++)
|
||||
{
|
||||
snapshot.Records[index] = ReadPersistedRecord(ref reader);
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = StoreSnapshotFieldCount; index < fieldCount; index++)
|
||||
{
|
||||
reader.Skip();
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
|
||||
{
|
||||
writer.WriteArrayHeader(PersistedRecordFieldCount);
|
||||
writer.Write(record.Key);
|
||||
writer.Write(record.SchemaId);
|
||||
writer.Write(record.StoredAt);
|
||||
writer.Write(record.Payload);
|
||||
}
|
||||
|
||||
private static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader)
|
||||
{
|
||||
var fieldCount = reader.ReadArrayHeader();
|
||||
var record = new CultPersistedRecord();
|
||||
|
||||
if (fieldCount > 0)
|
||||
{
|
||||
record.Key = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 1)
|
||||
{
|
||||
record.SchemaId = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 2)
|
||||
{
|
||||
record.StoredAt = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 3)
|
||||
{
|
||||
record.Payload = reader.ReadBytes()?.ToArray() ?? Array.Empty<byte>();
|
||||
}
|
||||
|
||||
for (var index = PersistedRecordFieldCount; index < fieldCount; index++)
|
||||
{
|
||||
reader.Skip();
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
|
||||
{
|
||||
writer.WriteArrayHeader(SchemaCatalogEntryFieldCount);
|
||||
writer.Write(entry.SchemaId);
|
||||
writer.Write(entry.SchemaName);
|
||||
writer.Write(entry.SchemaVersion);
|
||||
writer.Write(entry.ContentHash);
|
||||
writer.Write(entry.CanonicalSchemaJson);
|
||||
writer.WriteArrayHeader(entry.CompatibleSchemaIds.Length);
|
||||
foreach (var schemaId in entry.CompatibleSchemaIds)
|
||||
{
|
||||
writer.Write(schemaId);
|
||||
}
|
||||
}
|
||||
|
||||
private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader)
|
||||
{
|
||||
var fieldCount = reader.ReadArrayHeader();
|
||||
var entry = new CultSchemaCatalogEntry();
|
||||
|
||||
if (fieldCount > 0)
|
||||
{
|
||||
entry.SchemaId = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 1)
|
||||
{
|
||||
entry.SchemaName = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 2)
|
||||
{
|
||||
entry.SchemaVersion = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 3)
|
||||
{
|
||||
entry.ContentHash = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 4)
|
||||
{
|
||||
entry.CanonicalSchemaJson = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
|
||||
if (fieldCount > 5)
|
||||
{
|
||||
var compatibleCount = reader.ReadArrayHeader();
|
||||
entry.CompatibleSchemaIds = new string[compatibleCount];
|
||||
for (var index = 0; index < compatibleCount; index++)
|
||||
{
|
||||
entry.CompatibleSchemaIds[index] = reader.ReadString() ?? string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = SchemaCatalogEntryFieldCount; index < fieldCount; index++)
|
||||
{
|
||||
reader.Skip();
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
|
||||
public class SingleFileMessagePackBackingStore : SingleFileBackingStore
|
||||
|
|
@ -77,12 +290,12 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
|
|||
|
||||
protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Serialize(snapshot);
|
||||
return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
|
||||
}
|
||||
|
||||
protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Deserialize<CultPersistedStoreSnapshot>(data);
|
||||
return CultDocumentMessagePackSerialization.DeserializeSnapshot(data);
|
||||
}
|
||||
|
||||
protected override byte[] SerializePayload(object document)
|
||||
|
|
@ -104,22 +317,22 @@ public class MultiFileMessagePackBackingStore : MultiFileBackingStore
|
|||
|
||||
protected override byte[] SerializeRecord(CultPersistedRecord record)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Serialize(record);
|
||||
return CultDocumentMessagePackSerialization.SerializePersistedRecord(record);
|
||||
}
|
||||
|
||||
protected override CultPersistedRecord DeserializeRecord(byte[] data)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Deserialize<CultPersistedRecord>(data);
|
||||
return CultDocumentMessagePackSerialization.DeserializePersistedRecord(data);
|
||||
}
|
||||
|
||||
protected override byte[] SerializeCatalog(CultSchemaCatalogEntry[] catalog)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Serialize(catalog);
|
||||
return CultDocumentMessagePackSerialization.SerializeSchemaCatalog(catalog);
|
||||
}
|
||||
|
||||
protected override CultSchemaCatalogEntry[] DeserializeCatalog(byte[] data)
|
||||
{
|
||||
return CultDocumentMessagePackSerialization.Deserialize<CultSchemaCatalogEntry[]>(data);
|
||||
return CultDocumentMessagePackSerialization.DeserializeSchemaCatalog(data);
|
||||
}
|
||||
|
||||
protected override byte[] SerializePayload(object document)
|
||||
|
|
|
|||
|
|
@ -129,6 +129,45 @@ namespace GameCult.Caching.Tests
|
|||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SingleFileMessagePackBackingStore_RoundTrips_DiscoveredEntryType()
|
||||
{
|
||||
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack");
|
||||
|
||||
try
|
||||
{
|
||||
var writeStore = new SingleFileMessagePackBackingStore(filePath);
|
||||
var writeCache = new CultCache();
|
||||
writeCache.AddBackingStore(writeStore);
|
||||
|
||||
var entry = new NamedTestEntry
|
||||
{
|
||||
Name = "SingleFileMsgpack",
|
||||
Value = "payload"
|
||||
};
|
||||
|
||||
var handle = await writeCache.AddAsync(entry);
|
||||
writeStore.PushAll();
|
||||
|
||||
var readStore = new SingleFileMessagePackBackingStore(filePath);
|
||||
var readCache = new CultCache();
|
||||
readCache.AddBackingStore(readStore);
|
||||
await readCache.PullAllBackingStoresAsync();
|
||||
|
||||
var loaded = readCache.Get<NamedTestEntry>(handle.Key);
|
||||
|
||||
Assert.That(loaded, Is.Not.Null);
|
||||
Assert.That(loaded!.Value, Is.EqualTo("payload"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
File.Delete(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MessagePackSerialization_RoundTrips_CultRecordRef()
|
||||
{
|
||||
|
|
@ -195,6 +234,52 @@ namespace GameCult.Caching.Tests
|
|||
Assert.That(roundTrip.Value, Is.EqualTo("slot-array"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MessagePackStoreSerialization_RoundTrips_Snapshot_Record_And_Catalog()
|
||||
{
|
||||
var record = new CultPersistedRecord
|
||||
{
|
||||
Key = "record-1",
|
||||
SchemaId = "schema-1",
|
||||
StoredAt = "2026-05-08T12:00:00Z",
|
||||
Payload = new byte[] { 0x91, 0xA3, 0x66, 0x6F, 0x6F }
|
||||
};
|
||||
var catalog = new[]
|
||||
{
|
||||
new CultSchemaCatalogEntry
|
||||
{
|
||||
SchemaId = "schema-1",
|
||||
SchemaName = "tests.named_entry",
|
||||
SchemaVersion = "tests.named_entry.v1",
|
||||
ContentHash = "hash-1",
|
||||
CanonicalSchemaJson = "{\"fields\":2}",
|
||||
CompatibleSchemaIds = new[] { "schema-1", "schema-0" }
|
||||
}
|
||||
};
|
||||
var snapshot = new CultPersistedStoreSnapshot
|
||||
{
|
||||
FormatVersion = "cultcache.store.v1",
|
||||
SchemaCatalog = catalog,
|
||||
Records = new[] { record }
|
||||
};
|
||||
|
||||
var roundTripRecord = CultDocumentMessagePackSerialization.DeserializePersistedRecord(
|
||||
CultDocumentMessagePackSerialization.SerializePersistedRecord(record));
|
||||
var roundTripCatalog = CultDocumentMessagePackSerialization.DeserializeSchemaCatalog(
|
||||
CultDocumentMessagePackSerialization.SerializeSchemaCatalog(catalog));
|
||||
var roundTripSnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(
|
||||
CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
|
||||
|
||||
Assert.That(roundTripRecord.Key, Is.EqualTo("record-1"));
|
||||
Assert.That(roundTripRecord.SchemaId, Is.EqualTo("schema-1"));
|
||||
Assert.That(roundTripRecord.StoredAt, Is.EqualTo("2026-05-08T12:00:00Z"));
|
||||
Assert.That(roundTripRecord.Payload, Is.EqualTo(record.Payload));
|
||||
Assert.That(roundTripCatalog.Single().CompatibleSchemaIds, Is.EqualTo(catalog.Single().CompatibleSchemaIds));
|
||||
Assert.That(roundTripSnapshot.FormatVersion, Is.EqualTo("cultcache.store.v1"));
|
||||
Assert.That(roundTripSnapshot.SchemaCatalog.Single().SchemaName, Is.EqualTo("tests.named_entry"));
|
||||
Assert.That(roundTripSnapshot.Records.Single().Key, Is.EqualTo("record-1"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Registry_CanonicalSchemaJson_Tracks_Reference_Metadata()
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue