diff --git a/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs b/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs index 698b07ec..e5f9d5fd 100644 --- a/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs +++ b/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Text.Json; using GameCult.Caching; +using MessagePack; var records = 5000; var emitJson = false; @@ -95,10 +96,12 @@ internal sealed record BenchmarkMetric(string Name, int Operations, double Elaps [CultDocument("bench.item", "bench.item.v1")] internal sealed class BenchItem { - [CultName] + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public string Category { get; set; } = string.Empty; + [Key(2)] public int Value { get; set; } } diff --git a/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs b/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs index 2009b381..d7ed451e 100644 --- a/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs +++ b/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs @@ -287,7 +287,7 @@ public sealed class CultDocumentMessagePackGenerator : IIncrementalGenerator builder.AppendLine($" private static byte[] {document.PayloadSerializerMethodName}(object document)"); builder.AppendLine(" {"); builder.AppendLine($" var typed = ({document.DocumentTypeName})document;"); - builder.AppendLine(" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;"); + builder.AppendLine($" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.OptionsFor(typeof({document.DocumentTypeName}).Assembly);"); builder.AppendLine(" var buffer = new global::System.Buffers.ArrayBufferWriter();"); builder.AppendLine(" var writer = new global::MessagePack.MessagePackWriter(buffer);"); builder.AppendLine($" writer.WriteArrayHeader({document.Members.Length});"); @@ -305,7 +305,7 @@ public sealed class CultDocumentMessagePackGenerator : IIncrementalGenerator { builder.AppendLine($" private static object {document.PayloadDeserializerMethodName}(byte[] payload)"); builder.AppendLine(" {"); - builder.AppendLine(" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;"); + builder.AppendLine($" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.OptionsFor(typeof({document.DocumentTypeName}).Assembly);"); builder.AppendLine(" var reader = new global::MessagePack.MessagePackReader(payload);"); builder.AppendLine(" var count = reader.ReadArrayHeader();"); builder.AppendLine($" var value = new {document.DocumentTypeName}();"); diff --git a/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs b/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs new file mode 100644 index 00000000..038b65f6 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace GameCult.Caching.MessagePack; + +// The resolver type exposes a public static Instance of IFormatterResolver or a public parameterless constructor. +[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] +public sealed class CultCacheFormatterResolverAttribute : Attribute +{ + public CultCacheFormatterResolverAttribute(Type resolverType) + { + ResolverType = resolverType ?? throw new ArgumentNullException(nameof(resolverType)); + } + + public Type ResolverType { get; } +} diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 223d0310..b2e40fef 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -1,5 +1,8 @@ using System; using System.Buffers; +using System.Collections.Concurrent; +using System.Linq; +using System.Reflection; using GameCult.Caching; using MessagePack; using MessagePack.Formatters; @@ -32,12 +35,39 @@ public static class CultDocumentMessagePackSerialization private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; - public static readonly MessagePackSerializerOptions Options = - MessagePackSerializerOptions.Standard + public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty()); + + private static readonly ConcurrentDictionary AssemblyOptions = new(); + + public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) + { + if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly)); + return AssemblyOptions.GetOrAdd(documentAssembly, static assembly => + { + var resolvers = assembly.GetCustomAttributes() + .Select(attribute => CreateResolver(attribute.ResolverType)) + .ToArray(); + return resolvers.Length == 0 ? Options : Compose(resolvers); + }); + } + + private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers) + { + return MessagePackSerializerOptions.Standard .WithResolver(CompositeResolver.Create( - CultDocumentResolver.Instance, - StandardResolver.Instance)) - .WithSecurity(MessagePackSecurity.UntrustedData); + consumerResolvers.Append(CultDocumentResolver.Instance).Append(StandardResolver.Instance).ToArray())) + .WithSecurity(CultMessagePackSecurity.Instance); + } + + private static IFormatterResolver CreateResolver(Type resolverType) + { + var instance = resolverType.GetField("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null) + ?? resolverType.GetProperty("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null) + ?? (resolverType.GetConstructor(Type.EmptyTypes) != null ? Activator.CreateInstance(resolverType) : null); + return instance as IFormatterResolver + ?? throw new InvalidOperationException( + $"Formatter resolver {resolverType.FullName} needs a public static Instance or a public parameterless constructor and must implement {nameof(IFormatterResolver)}."); + } public static byte[] Serialize(T value) { @@ -66,7 +96,7 @@ public static class CultDocumentMessagePackSerialization } } - return MessagePackSerializer.Serialize(type, value, Options); + return MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly)); } public static object DeserializeUntyped(Type type, byte[] payload) @@ -83,7 +113,7 @@ public static class CultDocumentMessagePackSerialization return descriptor.GeneratedPayloadDeserializer(payload); } - return MessagePackSerializer.Deserialize(type, payload, Options) + return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly)) ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } diff --git a/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs b/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs new file mode 100644 index 00000000..64b237d5 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using MessagePack; + +namespace GameCult.Caching.MessagePack; + +// UntrustedData, plus a collision-resistant comparer for CultRecordRef dictionary keys over the key string. +public sealed class CultMessagePackSecurity : MessagePackSecurity +{ + public static readonly CultMessagePackSecurity Instance = new(UntrustedData); + + private CultMessagePackSecurity(MessagePackSecurity copyFrom) : base(copyFrom) { } + + protected override IEqualityComparer GetHashCollisionResistantEqualityComparer() + { + var type = typeof(T); + if (!type.IsGenericType || type.GetGenericTypeDefinition() != typeof(CultRecordRef<>)) + return base.GetHashCollisionResistantEqualityComparer(); + var comparerType = typeof(RecordRefComparer<>).MakeGenericType(type.GetGenericArguments()[0]); + return (IEqualityComparer)Activator.CreateInstance(comparerType, GetEqualityComparer())!; + } + + protected override MessagePackSecurity Clone() => new CultMessagePackSecurity(this); + + private sealed class RecordRefComparer : IEqualityComparer> + { + private readonly IEqualityComparer _keys; + + public RecordRefComparer(IEqualityComparer keys) => _keys = keys; + + public bool Equals(CultRecordRef x, CultRecordRef y) => _keys.Equals(x.Key.Value, y.Key.Value); + + public int GetHashCode(CultRecordRef value) => _keys.GetHashCode(value.Key.Value); + } +} diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..d6821542 --- /dev/null +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -0,0 +1,43 @@ +using GameCult.Caching.MessagePack; +using GameCult.Caching.Tests; +using MessagePack; +using MessagePack.Formatters; + +[assembly: CultCacheFormatterResolver(typeof(PairResolver))] + +namespace GameCult.Caching.Tests +{ + public struct Pair + { + public float A; + public float B; + } + + public sealed class PairResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new PairResolver(); + + public IMessagePackFormatter? GetFormatter() + { + return typeof(T) == typeof(Pair) ? (IMessagePackFormatter)(object)PairFormatter.Instance : null; + } + + private sealed class PairFormatter : IMessagePackFormatter + { + public static readonly PairFormatter Instance = new(); + + public void Serialize(ref MessagePackWriter writer, Pair value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(2); + writer.Write(value.A); + writer.Write(value.B); + } + + public Pair Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + if (reader.ReadArrayHeader() != 2) throw new MessagePackSerializationException("Pair is a two-element array."); + return new Pair { A = reader.ReadSingle(), B = reader.ReadSingle() }; + } + } + } +} diff --git a/tests/GameCult.Caching.Tests/SerializationOptionsTests.cs b/tests/GameCult.Caching.Tests/SerializationOptionsTests.cs new file mode 100644 index 00000000..476e0485 --- /dev/null +++ b/tests/GameCult.Caching.Tests/SerializationOptionsTests.cs @@ -0,0 +1,134 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + public class SerializationOptionsTests + { + // Captured on CultLib 825b4f7, before per-assembly options. + private const string InteropNotePayloadHex = + "96B963756C7463616368652E696E7465726F705F6E6F74652E7631AB6E6F74653A637368617270A6637368617270BD6373686172702077726F746520612043756C744361636865206E6F7465D9245468652076312073746F726520666F726D61742069732074686520636F6E74726163742E92A6637368617270A7696E7465726F70"; + + [Test] + public void InteropNoteBytesUnchanged() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(OptionsInteropNote) }); + var note = new OptionsInteropNote + { + DocumentId = "note:csharp", + AuthorRuntimeId = "csharp", + Title = "csharp wrote a CultCache note", + Body = "The v1 store format is the contract.", + Tags = new[] { "csharp", "interop" } + }; + + var payload = CultDocumentMessagePackSerialization.SerializeUntyped(note, typeof(OptionsInteropNote), registry); + + Assert.That(Convert.ToHexString(payload), Is.EqualTo(InteropNotePayloadHex)); + } + + [Test] + public async Task RefKeyedDictionaryRoundTrips() + { + var path = Path.Combine(Path.GetTempPath(), $"cultlib-refkeyed-{Guid.NewGuid():N}.cc"); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RefKeyedHolder) }); + var alpha = new CultRecordRef(new CultRecordKey("alpha")); + var beta = new CultRecordRef(new CultRecordKey("beta")); + try + { + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.UpsertAsync( + new RefKeyedHolder { Name = "holder", Weights = new() { [alpha] = 0.25f, [beta] = 2f } }, + new CultRecordHandle(new CultRecordKey("holder"))); + cache.FlushAllBackingStores(); + } + + using var reopened = new CultCache(registry); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); + var loaded = reopened.Get(new CultRecordKey("holder")); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded!.Weights[new CultRecordRef(new CultRecordKey("alpha"))], Is.EqualTo(0.25f)); + Assert.That(loaded.Weights[new CultRecordRef(new CultRecordKey("beta"))], Is.EqualTo(2f)); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Test] + public void DeclaredResolverEncodesValueTypeAsPositionalArray() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(PairHolder) }); + var holder = new PairHolder { Name = "pair", Value = new Pair { A = 1.5f, B = -2f } }; + Assert.That(registry.GetRequired().GeneratedPayloadSerializer, Is.Not.Null, "no generated codec"); + + var generated = CultDocumentMessagePackSerialization.SerializeUntyped(holder, typeof(PairHolder), registry); + // SerializeUntyped's reflective fallback, which a document without a generated codec takes. + var reflective = MessagePackSerializer.Serialize( + typeof(PairHolder), holder, CultDocumentMessagePackSerialization.OptionsFor(typeof(PairHolder).Assembly)); + + Assert.That(reflective, Is.EqualTo(generated)); + var reader = new MessagePackReader(generated); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); + reader.Skip(); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); + Assert.That(reader.ReadSingle(), Is.EqualTo(1.5f)); + Assert.That(reader.ReadSingle(), Is.EqualTo(-2f)); + + var fromGenerated = (PairHolder)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(PairHolder), generated, registry); + var fromReflective = (PairHolder)MessagePackSerializer.Deserialize( + typeof(PairHolder), reflective, CultDocumentMessagePackSerialization.OptionsFor(typeof(PairHolder).Assembly))!; + Assert.That(fromGenerated.Value, Is.EqualTo(holder.Value)); + Assert.That(fromReflective.Value, Is.EqualTo(holder.Value)); + } + + [Test] + public void UnknownAssemblyGetsBaseOptions() + { + Assert.That(CultDocumentMessagePackSerialization.OptionsFor(typeof(object).Assembly), + Is.SameAs(CultDocumentMessagePackSerialization.Options)); + Assert.That(CultDocumentMessagePackSerialization.Options.Resolver.GetFormatter(), Is.Null); + Assert.That(CultDocumentMessagePackSerialization.OptionsFor(typeof(Pair).Assembly), + Is.SameAs(CultDocumentMessagePackSerialization.OptionsFor(typeof(Pair).Assembly))); + Assert.That(CultDocumentMessagePackSerialization.OptionsFor(typeof(Pair).Assembly).Resolver.GetFormatter(), Is.Not.Null); + } + + [CultDocument("tests.pair_holder", "tests.pair_holder.v1")] + [MessagePackObject] + public sealed class PairHolder + { + [Key(0)] public string Name = string.Empty; + [Key(1)] public Pair Value; + } + + [CultDocument("cultcache.interop-note", "cultcache.interop_note.v1")] + public sealed class OptionsInteropNote + { + [Key(0)] public string SchemaVersion { get; set; } = "cultcache.interop_note.v1"; + [Key(1)] [CultName] public string DocumentId { get; set; } = string.Empty; + [Key(2)] public string AuthorRuntimeId { get; set; } = string.Empty; + [Key(3)] public string Title { get; set; } = string.Empty; + [Key(4)] public string Body { get; set; } = string.Empty; + [Key(5)] public string[] Tags { get; set; } = Array.Empty(); + } + + [CultDocument("tests.ref_keyed_holder", "tests.ref_keyed_holder.v1")] + public sealed class RefKeyedHolder + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public Dictionary, float> Weights = new(); + } + } +}