#nullable enable using System; using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Threading; using GameCult.Caching; using GameCult.Caching.MessagePack; using MessagePack; using NUnit.Framework; namespace GameCult.Caching.Tests { // The recursive-union golden and the arm refusal goldens in tests/vectors/union-parity. The C# reference is the only // writer (CULTCACHE_WRITE_VECTORS=1); every other run reads the committed bytes and compares them byte for byte, so a // vector is never regenerated by an ordinary run. Other runtimes are held to these bytes (docs/cultcache-union-parity-map.md U11). public class UnionParityVectorTests { private const string WriteVariable = "CULTCACHE_WRITE_VECTORS"; private const string Canary = "CANARY-union-golden-4c2a"; // The stored-at of the one record in union-surface.cc: the store's clock is pinned to it, so the store reproduces. private static readonly DateTimeOffset StoredAt = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(UnionSurface) }); private static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1"; // ---- the vector directory: written only under the write variable, otherwise read and compared ---- private static string VectorDirectory() { for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) { var vectors = Path.Combine(dir.FullName, "tests", "vectors"); if (Directory.Exists(vectors)) return Path.Combine(vectors, "union-parity"); } throw new DirectoryNotFoundException($"tests/vectors not found above {TestContext.CurrentContext.TestDirectory}."); } private static string VectorPath(string name) => Path.Combine(VectorDirectory(), name); // Under the write variable the reference's bytes become the file; otherwise they must equal the committed file. private static void Check(string name, byte[] reference) { var path = VectorPath(name); if (Writing) { Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllBytes(path, reference); return; } Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}"); } private static byte[] Committed(string name) { var path = VectorPath(name); Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); return File.ReadAllBytes(path); } // ---- the fixture: a recursive union, id-only arms, a union inside a record inside a union, a list of unions ---- // Keys 0, 1, 2, 5 and 6: 3 and 4 are retired, so the key map is not sequential. Wrap is the +2 nesting step beside // Box's +3, so any container depth from 4 up is reachable exactly. [Union(0, typeof(Box)), Union(1, typeof(Text)), Union(2, typeof(Use)), Union(5, typeof(Image)), Union(6, typeof(Wrap))] public abstract class Node { } [MessagePackObject] public sealed class Wrap : Node { [Key(0)] public Node? Inner; [Key(1)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Box : Node { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Children = new(); } [MessagePackObject] public sealed class Text : Node { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public string Content = string.Empty; } [MessagePackObject] public sealed class Use : Node { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public Composition Composition = new(); [Key(2)] public List Slots = new(); } // Slot 2 is a retired gap, written nil. [MessagePackObject] public sealed class Image : Node { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public string Uri = string.Empty; [Key(3)] public string Alt = string.Empty; } // A record, not an arm, holding a union. [MessagePackObject] public sealed class Composition { [Key(0)] public string Name = string.Empty; [Key(1)] public Node? Body; } // Floor and Ceiling are id-only arms at the ends of the Int32 key domain. [Union(0, typeof(Pending)), Union(1, typeof(Completed)), Union(2, typeof(Denied)), Union(int.MinValue, typeof(Floor)), Union(int.MaxValue, typeof(Ceiling))] public abstract class Outcome { } [MessagePackObject] public sealed class Pending : Outcome { [Key(0)] public string Message = string.Empty; [Key(1)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Completed : Outcome { [Key(0)] public int SourceVersion; [Key(1)] [CultElementId] public string Id = string.Empty; } // The arms of a list carry an id, so Denied, Floor and Ceiling are written [key, [id]], not as unit arms. [MessagePackObject] public sealed class Denied : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Floor : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Ceiling : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } // Item names Tool and Part under keys 1 and 2; CraftedItem, the inner union both derive from, names them under 0 and 1. [Union(0, typeof(Gem)), Union(1, typeof(Tool)), Union(2, typeof(Part))] public abstract class Item { } [Union(0, typeof(Tool)), Union(1, typeof(Part))] public abstract class CraftedItem : Item { } [MessagePackObject] public sealed class Gem : Item { [Key(0)] public string Name = string.Empty; } [MessagePackObject] public sealed class Tool : CraftedItem { [Key(0)] public string Name = string.Empty; [Key(1)] public int Uses; } [MessagePackObject] public sealed class Part : CraftedItem { [Key(0)] public string Name = string.Empty; [Key(1)] public int Count; } [CultDocument("tests.union_surface", "tests.union_surface.v1")] [MessagePackObject] public sealed class UnionSurface { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public int Version; [Key(2)] public Node? Root; [Key(3)] public List Outcomes = new(); [Key(4)] public Item? Part; [Key(5)] public CraftedItem? Crafted; [Key(6)] public Node? Missing; } private static UnionSurface Constructed() => Constructed(null); private static UnionSurface Constructed(Node? root) => new() { Id = "surface", Version = 3, Root = root ?? new Box { Id = "0000000000c0", Children = { new Text { Id = "0000000000c1", Content = "hello" }, new Use { Id = "0000000000c2", Composition = new Composition { Name = "comp", Body = new Box { Id = "0000000000d0", Children = { new Text { Id = "0000000000d1", Content = "x" }, new Image { Id = "0000000000d2", Uri = "uri://a", Alt = "alt" } } } }, Slots = { new Text { Id = "0000000000e1", Content = "slot" }, null!, new Box { Id = "0000000000e2" } } }, new Image { Id = "0000000000c3", Uri = "uri://b", Alt = "" } } }, Outcomes = { new Pending { Message = "wait", Id = "0000000000a1" }, new Completed { SourceVersion = 7, Id = "0000000000a2" }, new Denied { Id = "0000000000a3" } }, Part = new Tool { Name = "hammer", Uses = 12 }, Crafted = new Part { Name = "gear", Count = 4 }, Missing = null }; // A readable tree of a decoded value, for comparing what was read without re-encoding it. private static string Describe(object? value) => value switch { null => "nil", UnionSurface s => $"Surface({s.Id},{s.Version},{Describe(s.Root)},[{string.Join(";", s.Outcomes.Select(Describe))}],{Describe(s.Part)},{Describe(s.Crafted)},{Describe(s.Missing)})", Box b => $"Box({b.Id},[{string.Join(";", b.Children.Select(Describe))}])", Text t => $"Text({t.Id},{t.Content})", Use u => $"Use({u.Id},{Describe(u.Composition)},[{string.Join(";", u.Slots.Select(Describe))}])", Image i => $"Image({i.Id},{i.Uri},{i.Alt})", Composition c => $"Composition({c.Name},{Describe(c.Body)})", Pending p => $"Pending({p.Message})", Completed c => $"Completed({c.SourceVersion})", Denied => "Denied", Floor => "Floor", Ceiling => "Ceiling", Wrap w => $"Wrap({Describe(w.Inner)})", Gem g => $"Gem({g.Name})", Tool t => $"Tool({t.Name},{t.Uses})", Part p => $"Part({p.Name},{p.Count})", _ => throw new InvalidOperationException(value.GetType().Name) }; // ---- the hand-written emitter: the expected bytes, written with MessagePackWriter, not captured from the code under test ---- private delegate void Emit(ref MessagePackWriter writer); private static byte[] Raw(Emit emit) { var buffer = new ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); emit(ref writer); writer.Flush(); return buffer.WrittenSpan.ToArray(); } private static readonly Emit Nil = (ref MessagePackWriter w) => w.WriteNil(); private static readonly Emit ExtraSlot = (ref MessagePackWriter w) => w.Write("extra-slot"); private static Emit Str(string s) => (ref MessagePackWriter w) => w.Write(s); private static Emit Num(int n) => (ref MessagePackWriter w) => w.Write(n); private static Emit Slots(params Emit[] slots) => (ref MessagePackWriter w) => { w.WriteArrayHeader(slots.Length); foreach (var slot in slots) slot(ref w); }; private static Emit ArmOf(int key, params Emit[] slots) => (ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write(key); Slots(slots)(ref w); }; // The slots, with one slot past the last declared key when extra. private static Emit Plus(bool extra, params Emit[] slots) => Slots(extra ? slots.Append(ExtraSlot).ToArray() : slots); private sealed record Extra(bool Record = false, bool Composition = false, bool RootArm = false, bool ListElement = false); private static Emit BoxArm(string id, Emit children, bool extra = false) => ArmOf(0, extra ? new[] { Str(id), children, ExtraSlot } : new[] { Str(id), children }); private static Emit TextArm(string id, string content, bool extra = false) => ArmOf(1, extra ? new[] { Str(id), Str(content), ExtraSlot } : new[] { Str(id), Str(content) }); private static Emit ImageArm(string id, string uri, string alt) => ArmOf(5, Str(id), Str(uri), Nil, Str(alt)); // The constructed value's payload, with a part of the structure replaceable to build a refusal or an extra-slot case. private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, Emit? trailing = null) { extra ??= new Extra(); root ??= BoxArm("0000000000c0", Slots( TextArm("0000000000c1", "hello"), ArmOf(2, Str("0000000000c2"), Plus(extra.Composition, Str("comp"), BoxArm("0000000000d0", Slots(TextArm("0000000000d1", "x"), ImageArm("0000000000d2", "uri://a", "alt")))), Slots(TextArm("0000000000e1", "slot", extra.ListElement), Nil, BoxArm("0000000000e2", Slots()))), ImageArm("0000000000c3", "uri://b", "")), extra.RootArm); outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3"))); crafted ??= ArmOf(1, Str("gear"), Num(4)); var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil }; return Raw(Slots(trailing != null ? slots.Append(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots)); } private static byte[] SerializeValue(UnionSurface value) => CultDocumentMessagePackSerialization.SerializeUntyped(value, typeof(UnionSurface), Registry); private static UnionSurface Decode(byte[] payload) => (UnionSurface)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(UnionSurface), payload, Registry); // ---- the refusal and extra-slot vectors, each with its expected outcome as data ---- // file, outcome ("refused" | "read-only" | "accepted"), union (logical name), arm key (null for none), detail. private sealed record Case(string File, string Outcome, string Union, int? Key, string Detail, byte[] Payload); private static IReadOnlyList Cases() => new[] { new Case("unknown-arm-field.msgpack", "refused", "Node", 3, "unknown-arm", Surface(root: ArmOf(3, Str("r"), Str(Canary)))), new Case("unknown-arm-list.msgpack", "refused", "Outcome", 7, "unknown-arm", Surface(outcomes: Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(7, Str(Canary))))), new Case("unknown-arm-nested.msgpack", "refused", "Node", 9, "unknown-arm", Surface(root: BoxArm("0000000000c0", Slots(BoxArm("0000000000d3", Slots(BoxArm("0000000000d4", Slots(ArmOf(9, Str("deep"), Str(Canary)))))))))), new Case("inner-union-outer-key.msgpack", "refused", "CraftedItem", 2, "unknown-arm", Surface(crafted: ArmOf(2, Str("gear"), Num(4)))), new Case("arm-arity.msgpack", "refused", "Node", 1, "arity", Surface(root: (ref MessagePackWriter w) => { w.WriteArrayHeader(1); w.Write(1); })), new Case("extra-slot-record.msgpack", "read-only", "UnionSurface", null, "record", Surface(extra: new Extra(Record: true))), new Case("extra-slot-nested-record.msgpack", "read-only", "Composition", null, "nested-record", Surface(extra: new Extra(Composition: true))), new Case("extra-slot-arm.msgpack", "read-only", "Node", 0, "arm", Surface(extra: new Extra(RootArm: true))), new Case("extra-slot-list-element.msgpack", "read-only", "Node", 1, "list-element", Surface(extra: new Extra(ListElement: true))), }; private static byte[] OutcomesFile() => Raw((ref MessagePackWriter w) => { var cases = Cases(); w.WriteArrayHeader(cases.Count); foreach (var c in cases) { w.WriteArrayHeader(5); w.Write(c.File); w.Write(c.Outcome); w.Write(c.Union); if (c.Key is { } key) w.Write(key); else w.WriteNil(); w.Write(c.Detail); } }); private static string UnionName(string logical) => logical switch { "Node" => typeof(Node).FullName!, "Outcome" => typeof(Outcome).FullName!, "CraftedItem" => typeof(CraftedItem).FullName!, "UnionSurface" => typeof(UnionSurface).FullName!, _ => throw new ArgumentOutOfRangeException(nameof(logical), logical, null) }; private static string Chain(Exception exception) { var parts = new List(); for (var cause = exception; cause != null; cause = cause.InnerException) parts.Add(cause.Message); return string.Join(" <- ", parts); } // ---- unit arms: a union no list reaches, so its arms carry no element id and a member-less arm is a unit arm ---- [Union(0, typeof(Waiting)), Union(2, typeof(Denied2)), Union(int.MinValue, typeof(Floor2)), Union(int.MaxValue, typeof(Ceiling2))] public abstract class UnitOutcome { } [MessagePackObject] public sealed class Waiting : UnitOutcome { [Key(0)] public string Message = string.Empty; } [MessagePackObject] public sealed class Denied2 : UnitOutcome { } [MessagePackObject] public sealed class Floor2 : UnitOutcome { } [MessagePackObject] public sealed class Ceiling2 : UnitOutcome { } private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly); // The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes. [Test] public void UnitArmVectorsAreWhatTheReferenceWrites() { var arms = new (string File, UnitOutcome Value, int Key, string Name)[] { ("unit-arm-denied.msgpack", new Denied2(), 2, nameof(Denied2)), ("unit-arm-floor.msgpack", new Floor2(), int.MinValue, nameof(Floor2)), ("unit-arm-ceiling.msgpack", new Ceiling2(), int.MaxValue, nameof(Ceiling2)), }; foreach (var (file, value, key, name) in arms) { var written = MessagePackSerializer.Serialize(value, UnitOptions); Assert.That(Convert.ToHexString(written), Is.EqualTo(Convert.ToHexString(Raw(ArmOf(key)))), $"{name}: a unit arm is [key, []]"); Check(file, written); var committed = Committed(file); var decoded = MessagePackSerializer.Deserialize(committed, UnitOptions); Assert.That(decoded.GetType().Name, Is.EqualTo(name), file); Assert.That(Convert.ToHexString(MessagePackSerializer.Serialize(decoded, UnitOptions)), Is.EqualTo(Convert.ToHexString(committed)), $"{file} re-encodes byte-identical"); } } // ---- tests ---- [Test] public void UnionSurfaceVectorMatches() { var value = Constructed(); var written = SerializeValue(value); Assert.That(Convert.ToHexString(written), Is.EqualTo(Convert.ToHexString(Surface())), "the hand-written bytes are what the reference writes"); Assert.That(Convert.ToHexString(MessagePackSerializer.Serialize(value, MessagePackSerializerOptions.Standard)), Is.EqualTo(Convert.ToHexString(written))); Check("union-surface.msgpack", written); var committed = Committed("union-surface.msgpack"); var decoded = Decode(committed); Assert.That(Describe(decoded), Is.EqualTo(Describe(value))); Assert.That(Convert.ToHexString(SerializeValue(decoded)), Is.EqualTo(Convert.ToHexString(committed)), "the committed payload re-encodes byte-identical"); var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}"); Directory.CreateDirectory(work); try { var scratch = Path.Combine(work, "union-surface.cc"); using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry, StoreFlushOnDispose = true })) { cache.Clock = () => StoredAt; cache.UpsertAsync(typeof(UnionSurface), value, new CultRecordKey("surface")).GetAwaiter().GetResult(); cache.FlushAsync().GetAwaiter().GetResult(); } Check("union-surface.cc", File.ReadAllBytes(scratch)); // The committed store: one record whose payload is the committed payload, holding the constructed value, and rewritten unchanged. var store = Committed("union-surface.cc"); var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(store); Assert.That(snapshot.Records, Has.Length.EqualTo(1)); Assert.That(Convert.ToHexString(snapshot.Records[0].Payload), Is.EqualTo(Convert.ToHexString(committed))); File.WriteAllBytes(scratch, store); using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry })) { Assert.That(Describe(cache.Get(new CultRecordKey("surface"))), Is.EqualTo(Describe(value))); cache.BackingStores[0].PushAll(); } Assert.That(Convert.ToHexString(File.ReadAllBytes(scratch)), Is.EqualTo(Convert.ToHexString(store))); } finally { Directory.Delete(work, recursive: true); } } // The value names every arm shape the vectors exist to hold other runtimes to. [Test] public void ConstructedValueExercisesEveryArmShape() { var value = Decode(Committed("union-surface.msgpack")); var root = (Box)value.Root!; var use = root.Children.OfType().Single(); var inner = (Box)use.Composition.Body!; Assert.Multiple(() => { Assert.That(value.Outcomes.OfType().Count(), Is.EqualTo(1), "an id-only arm"); Assert.That(root.Children, Has.Count.GreaterThan(1), "a list of unions"); Assert.That(use.Slots.Any(s => s is Box) && use.Slots.Any(s => s is null), Is.True, "a list of unions holding a nested union arm and nil"); Assert.That(inner.Children.OfType().Count(), Is.EqualTo(1), "a union inside a record inside a union, four arms deep"); Assert.That(value.Part, Is.TypeOf()); Assert.That(value.Crafted, Is.TypeOf(), "an inner union keyed apart from its outer"); Assert.That(value.Missing, Is.Null); }); } [Test] public void UnionRefusalVectorsRefused() { foreach (var c in Cases()) Check(c.File, c.Payload); Check("outcomes.msgpack", OutcomesFile()); // From here only the committed files speak: the outcomes file is the data the other runtimes read. var golden = Describe(Constructed()); var reader = new MessagePackReader(Committed("outcomes.msgpack")); var count = reader.ReadArrayHeader(); Assert.That(count, Is.EqualTo(Cases().Count)); for (var i = 0; i < count; i++) { Assert.That(reader.ReadArrayHeader(), Is.EqualTo(5)); var file = reader.ReadString()!; var outcome = reader.ReadString()!; var union = reader.ReadString()!; int? key = reader.TryReadNil() ? null : reader.ReadInt32(); var detail = reader.ReadString()!; var payload = Committed(file); if (outcome == "refused") { UnionSurface? accepted = null; Exception? thrown = null; try { accepted = Decode(payload); } catch (Exception exception) { thrown = exception; } Assert.That(thrown, Is.Not.Null, $"{file} was accepted"); var message = Chain(thrown!); Assert.That(message, Does.Not.Contain(Canary), $"{file}: a refusal never echoes the payload"); if (detail == "arity") { Assert.That(message, Does.Contain("Invalid Union data"), file); } else { Assert.That(message, Does.Contain(UnionName(union)).And.Contain($"key {key} "), file); Assert.That(accepted, Is.Null, file); } } else { Assert.That(outcome, Is.EqualTo("read-only"), file); // The record is read whole; what makes it read-only is the slot rule of the slot-tolerance cut, not asserted here. Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); } } } // ---- the bounds vectors: arm keys at the ends of the Int32 domain, nil arm slots, payload nesting ---- private const int Limit = CultMessagePackSecurity.MaximumPayloadDepth; // The outermost array is depth 1, a Wrap arm ([6, [inner]]) adds 2, a Box with no children 3, a Text leaf 2. // The at-limit and past-limit payloads are the same chain of Wrap arms; the leaf is what makes the depth odd or even. private const int AtLimitWraps = (Limit - 1 - 3) / 2; private const int PastLimitWraps = (Limit + 1 - 1 - 2) / 2; private static Emit RawKey(Emit key, params Emit[] slots) => (ref MessagePackWriter w) => { w.WriteArrayHeader(2); key(ref w); Slots(slots)(ref w); }; private static Emit WithOutcome(Emit element) => Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3")), element); private static Emit WrapChain(int wraps, Emit leaf) => wraps == 0 ? leaf : ArmOf(6, WrapChain(wraps - 1, leaf), Str("0000000000b1")); private static Node WrapChainValue(int wraps, Node leaf) { for (var i = 0; i < wraps; i++) leaf = new Wrap { Inner = leaf, Id = "0000000000b1" }; return leaf; } // n arrays nested inside one another around a single string. private static Emit Nested(int n) => n == 1 ? Slots(Str(Canary)) : Slots(Nested(n - 1)); private static Emit AtLimitLeaf => BoxArm("0000000000f0", Slots()); private static Emit PastLimitLeaf => TextArm("0000000000f0", Canary); private static IReadOnlyList BoundsCases() => new[] { new Case("arm-key-int32-min.msgpack", "accepted", "Outcome", int.MinValue, "arm-key-int32-min.msgpack", Surface(outcomes: WithOutcome(ArmOf(int.MinValue, Str("0000000000a4"))))), new Case("arm-key-int32-max.msgpack", "accepted", "Outcome", int.MaxValue, "arm-key-int32-max.msgpack", Surface(outcomes: WithOutcome(ArmOf(int.MaxValue, Str("0000000000a5"))))), new Case("arm-key-wide-coding.msgpack", "accepted", "Outcome", int.MaxValue, "arm-key-int32-max.msgpack", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.WriteUInt64(int.MaxValue), Str("0000000000a5"))))), new Case("arm-key-above-int32.msgpack", "refused", "Outcome", null, "key-out-of-range", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.Write(0x80000000u))))), new Case("arm-key-below-int32.msgpack", "refused", "Outcome", null, "key-out-of-range", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.Write(-2147483649L))))), new Case("arm-key-float32.msgpack", "refused", "Outcome", null, "key-not-integer", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.Write(1.0f), Num(7))))), new Case("arm-key-float64-negzero.msgpack", "refused", "Outcome", null, "key-not-integer", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.Write(-0.0), Str("wait"))))), new Case("nil-arm-slots.msgpack", "refused", "Outcome", 1, "nil-slots", Surface(outcomes: WithOutcome((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write(1); w.WriteNil(); }))), new Case("depth-at-limit.msgpack", "accepted", "UnionSurface", null, "depth-at-limit.msgpack", Surface(root: WrapChain(AtLimitWraps, AtLimitLeaf))), new Case("depth-past-limit.msgpack", "refused", "UnionSurface", null, "depth", Surface(root: WrapChain(PastLimitWraps, PastLimitLeaf))), new Case("depth-skipped-at-limit.msgpack", "read-only", "UnionSurface", null, "depth", Surface(trailing: Nested(Limit - 1))), new Case("depth-skipped-past-limit.msgpack", "refused", "UnionSurface", null, "depth", Surface(trailing: Nested(Limit))), }; private static byte[] BoundsOutcomesFile() => Raw((ref MessagePackWriter w) => { var cases = BoundsCases(); w.WriteArrayHeader(cases.Count + 1); w.WriteArrayHeader(2); w.Write("limit"); w.Write(Limit); foreach (var c in cases) { w.WriteArrayHeader(5); w.Write(c.File); w.Write(c.Outcome); w.Write(c.Union); if (c.Key is { } key) w.Write(key); else w.WriteNil(); w.Write(c.Detail); } }); // The container depth of a payload, counted with the reader's typed reads and an explicit stack: not the rule's code. private static int DepthOf(byte[] payload) { var reader = new MessagePackReader(payload); var open = new Stack(); var deepest = 0; while (true) { switch (reader.NextMessagePackType) { case MessagePackType.Array: case MessagePackType.Map: var isArray = reader.NextMessagePackType == MessagePackType.Array; long children = isArray ? reader.ReadArrayHeader() : 2L * reader.ReadMapHeader(); deepest = Math.Max(deepest, open.Count + 1); if (children > 0) { open.Push(children); continue; } break; case MessagePackType.Nil: reader.ReadNil(); break; case MessagePackType.Boolean: reader.ReadBoolean(); break; case MessagePackType.Integer: if (reader.NextCode == MessagePackCode.UInt64) reader.ReadUInt64(); else reader.ReadInt64(); break; case MessagePackType.Float: reader.ReadDouble(); break; case MessagePackType.String: reader.ReadString(); break; case MessagePackType.Binary: reader.ReadBytes(); break; case MessagePackType.Extension: reader.ReadExtensionFormat(); break; default: throw new InvalidOperationException(reader.NextMessagePackType.ToString()); } while (open.Count > 0) { var left = open.Pop() - 1; if (left > 0) { open.Push(left); break; } } if (open.Count == 0) return deepest; } } private static Exception Refusal(Action action) { try { action(); } catch (Exception exception) { return exception; } throw new AssertionException("the payload was accepted"); } [Test] public void BoundsOutcomesFromCommittedFiles() { foreach (var c in BoundsCases()) Check(c.File, c.Payload); Check("bounds-outcomes.msgpack", BoundsOutcomesFile()); // From here only the committed files speak. var golden = Describe(Constructed()); var reader = new MessagePackReader(Committed("bounds-outcomes.msgpack")); var count = reader.ReadArrayHeader(); Assert.That(count, Is.EqualTo(BoundsCases().Count + 1)); Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); Assert.That(reader.ReadString(), Is.EqualTo("limit")); Assert.That(reader.ReadInt32(), Is.EqualTo(CultMessagePackSecurity.MaximumPayloadDepth), "the limit entry states the reference's limit"); for (var i = 1; i < count; i++) { Assert.That(reader.ReadArrayHeader(), Is.EqualTo(5)); var file = reader.ReadString()!; var outcome = reader.ReadString()!; var union = reader.ReadString()!; int? key = reader.TryReadNil() ? null : reader.ReadInt32(); var detail = reader.ReadString()!; var payload = Committed(file); switch (outcome) { case "accepted": Assert.That(Convert.ToHexString(SerializeValue(Decode(payload))), Is.EqualTo(Convert.ToHexString(Committed(detail))), $"{file} re-encodes to {detail}"); break; case "read-only": Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past"); break; case "refused": var message = Chain(Refusal(() => Decode(payload))); Assert.That(message, Does.Not.Contain(Canary), $"{file}: a refusal never echoes the payload"); Assert.That(message, Does.Contain(UnionName(union)), file); switch (detail) { case "key-out-of-range": Assert.That(message, Does.Contain("outside Int32").And.Not.Contain("has no arm for key"), file); break; case "key-not-integer": Assert.That(message, Does.Contain("non-integer"), file); break; case "nil-slots": Assert.That(message, Does.Contain("nil slots").And.Contain($"arm {key} "), file); break; case "depth": Assert.That(message, Does.Contain($"nests {Limit + 1} MessagePack containers deep").And.Contain($"the limit is {Limit}").And.Contain("refused on read"), file); break; default: Assert.Fail($"{file}: unknown detail {detail}"); break; } break; default: Assert.Fail($"{file}: unknown outcome {outcome}"); break; } } } [Test] public void ArmKeyDomainVectors() { var min = Committed("arm-key-int32-min.msgpack"); var max = Committed("arm-key-int32-max.msgpack"); var wide = Committed("arm-key-wide-coding.msgpack"); Assert.That(Convert.ToHexString(min), Does.Contain("D280000000"), "Floor is written as an int32 key"); Assert.That(Convert.ToHexString(max), Does.Contain("CE7FFFFFFF"), "Ceiling is written as a uint32 key"); Assert.That(Convert.ToHexString(wide), Does.Contain("CF000000007FFFFFFF"), "the wide coding is a uint64 key"); Assert.That(Decode(min).Outcomes, Has.Count.EqualTo(4)); Assert.That(Decode(min).Outcomes[3], Is.TypeOf()); Assert.That(Decode(max).Outcomes[3], Is.TypeOf()); Assert.That(Decode(wide).Outcomes[3], Is.TypeOf(), "an unsigned 64-bit coding of an in-range key is the same key"); Assert.That(Convert.ToHexString(SerializeValue(Decode(wide))), Is.EqualTo(Convert.ToHexString(max)), "and re-encodes as the canonical coding"); Assert.That(Convert.ToHexString(SerializeValue(Decode(min))), Is.EqualTo(Convert.ToHexString(min))); Assert.That(Chain(Refusal(() => Decode(Committed("arm-key-above-int32.msgpack")))), Does.Contain("outside Int32").And.Contain(UnionName("Outcome"))); Assert.That(Chain(Refusal(() => Decode(Committed("arm-key-below-int32.msgpack")))), Does.Contain("outside Int32").And.Contain(UnionName("Outcome"))); Assert.That(Chain(Refusal(() => Decode(Committed("arm-key-float32.msgpack")))), Does.Contain("non-integer").And.Contain(UnionName("Outcome"))); Assert.That(Chain(Refusal(() => Decode(Committed("arm-key-float64-negzero.msgpack")))), Does.Contain("non-integer").And.Contain(UnionName("Outcome"))); } [Test] public void NilArmSlotsRefused() { var message = Chain(Refusal(() => Decode(Committed("nil-arm-slots.msgpack")))); Assert.That(message, Does.Contain(UnionName("Outcome")).And.Contain("arm 1 has nil slots")); } [Test] public void DepthAtLimitAcceptedBothWays() { var payload = Committed("depth-at-limit.msgpack"); Assert.That(DepthOf(payload), Is.EqualTo(Limit), "the vector nests exactly the limit, counted independently"); Assert.That(Convert.ToHexString(SerializeValue(Decode(payload))), Is.EqualTo(Convert.ToHexString(payload)), "read, then written again"); var built = Constructed(WrapChainValue(AtLimitWraps, new Box { Id = "0000000000f0" })); Assert.That(Convert.ToHexString(SerializeValue(built)), Is.EqualTo(Convert.ToHexString(payload)), "the constructed value writes the vector"); } [Test] public void DepthPastLimitRefusedBothWays() { var payload = Committed("depth-past-limit.msgpack"); Assert.That(DepthOf(payload), Is.EqualTo(Limit + 1), "one container past the limit, counted independently"); var read = Chain(Refusal(() => Decode(payload))); Assert.That(read, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"the limit is {Limit}").And.Contain("refused on read").And.Not.Contain(Canary)); byte[]? written = null; var write = Chain(Refusal(() => written = SerializeValue(Constructed(WrapChainValue(PastLimitWraps, new Text { Id = "0000000000f0", Content = Canary }))))); Assert.That(written, Is.Null, "nothing is returned"); Assert.That(write, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"the limit is {Limit}").And.Contain("refused on write").And.Not.Contain(Canary)); // A single-file store holding the payload fails to load through the same entry point. var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}"); Directory.CreateDirectory(work); try { var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(Committed("union-surface.cc")); snapshot.Records[0].Payload = payload; var scratch = Path.Combine(work, "deep.cc"); File.WriteAllBytes(scratch, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var load = Chain(Refusal(() => CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry }).Dispose())); Assert.That(load, Does.Contain("surface").And.Contain($"the limit is {Limit}").And.Not.Contain(Canary)); } finally { Directory.Delete(work, recursive: true); } } [Test] public void SkippedContainersCount() { var past = Committed("depth-skipped-past-limit.msgpack"); var atLimit = Committed("depth-skipped-at-limit.msgpack"); Assert.That(DepthOf(Committed("union-surface.msgpack")), Is.LessThan(20), "what the formatters read is shallow"); Assert.That(DepthOf(atLimit), Is.EqualTo(Limit)); Assert.That(DepthOf(past), Is.EqualTo(Limit + 1)); Assert.That(Describe(Decode(atLimit)), Is.EqualTo(Describe(Constructed())), "skipped content at the limit is read past"); var message = Chain(Refusal(() => Decode(past))); Assert.That(message, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"the limit is {Limit}").And.Not.Contain(Canary)); // MessagePack alone counts only what its formatters read: it accepts the payload the walk refuses. Assert.That(() => MessagePackSerializer.Deserialize(typeof(UnionSurface), past, CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly)), Throws.Nothing, "MessagePack's own counter never sees skipped containers"); } [Test] public void WalkIsIterative() { // 100000 nested arrays in an extra trailing slot, on a thread with a 256 KiB stack. var deep = Surface(trailing: (ref MessagePackWriter w) => { w.WriteRaw(Enumerable.Repeat((byte)0x91, 100_000).ToArray()); w.WriteArrayHeader(0); }); Exception? thrown = null; var thread = new Thread(() => { thrown = Refusal(() => Decode(deep)); }, 256 * 1024); thread.Start(); thread.Join(); Assert.That(thrown, Is.Not.Null); Assert.That(Chain(thrown!), Does.Contain($"the limit is {Limit}")); } // Runs whatever the switch reads when the test starts: it sets the switch itself, runs every test body that calls Check // under each off value, and writes only a probe file. The fingerprint is each file's bytes and write time, so a rewrite // with identical bytes shows. [Test, NonParallelizable] public void VectorsNotRewrittenByDefault() { string Fingerprint() => string.Join("|", Directory.EnumerateFiles(VectorDirectory()).OrderBy(f => f, StringComparer.Ordinal) .Select(f => Path.GetFileName(f) + "=" + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(f))) + "@" + File.GetLastWriteTimeUtc(f).Ticks)); const string probe = "gate-probe.msgpack"; var path = VectorPath(probe); var before = Fingerprint(); var inherited = Environment.GetEnvironmentVariable(WriteVariable); try { foreach (var off in new string?[] { null, string.Empty, "0", "true", "11" }) { Environment.SetEnvironmentVariable(WriteVariable, off); Assert.Throws(() => Check(probe, new byte[] { 0x90 }), $"a missing vector fails the run, switch {off ?? "unset"}"); Assert.That(File.Exists(path), Is.False, $"switch {off ?? "unset"} writes nothing"); File.WriteAllBytes(path, new byte[] { 0x91 }); Assert.Throws(() => Check(probe, new byte[] { 0x90 }), $"bytes that differ from the vector fail the run, switch {off ?? "unset"}"); Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x91 }), $"switch {off ?? "unset"} rewrites nothing"); File.Delete(path); foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles }) body(); } Environment.SetEnvironmentVariable(WriteVariable, "1"); Check(probe, new byte[] { 0x90 }); Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x90 }), "1 is the one way to write a vector"); } finally { Environment.SetEnvironmentVariable(WriteVariable, inherited); if (File.Exists(path)) File.Delete(path); } Assert.That(Fingerprint(), Is.EqualTo(before), "no committed vector changed"); } } }