using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.Json; using System.Threading.Tasks; using FluentAssertions; using GameCult.Caching; using GameCult.Caching.MessagePack; using GameCult.Networking; using MessagePack; using NUnit.Framework; namespace GameCult.Mesh.Tests; /// /// Content-plane parity vectors shared with packages/cultnet-rs/tests/content.rs. Each runtime /// writes its own file under contracts/cultmesh and judges both. The chunk answers in these /// files are produced by itself, so the failure /// spellings are the reference's own bytes. /// [TestFixture] public sealed class CultMeshContentVectorTests { private const int BodyLength = 1_315_551; private const int ChunkSize = 262_144; private const string CreatedAtUtc = "2026-09-22T00:00:00.0000000+00:00"; private const string ArtifactId = "eureka/content-vector"; private sealed record Vector(string Label, string Kind, byte[] Bytes); [Test] public void WriteContentVectors() { if (Environment.GetEnvironmentVariable("CULTNET_WRITE_VECTORS") != "1") Assert.Ignore("Set CULTNET_WRITE_VECTORS=1 to (re)write the committed vector file."); File.WriteAllText(VectorPath("content-vectors.cs-written.json"), ToJson(BuildVectors(out _, out _))); File.WriteAllText(VectorPath("cdn-record-key-vectors.json"), RecordKeyVectorsToJson(BuildRecordKeyVectors())); File.WriteAllText(VectorPath("content-acceptance-vectors.json"), AcceptanceVectorsToJson(BuildAcceptanceVectors())); } [Test] public void CommittedCsVectorsEqualAFreshEncode() { var fresh = BuildVectors(out _, out _); var committed = ReadVectors("content-vectors.cs-written.json"); committed.Select(v => (v.Label, v.Kind)).Should().Equal(fresh.Select(v => (v.Label, v.Kind))); foreach (var (expected, actual) in fresh.Zip(committed)) actual.Bytes.Should().Equal(expected.Bytes, $"vector '{actual.Label}' drifted from the reference's encode"); } [Test] public void RustVectorsDecodeInTheReference() { BuildVectors(out var cache, out var messages); var rust = ReadVectors("content-vectors.rs-written.json"); rust.Should().NotBeEmpty(); rust.Select(v => v.Label).Should().Equal(messages.Keys, "both runtimes write the same labelled vectors"); foreach (var vector in rust) { var expected = messages[vector.Label]; switch (vector.Kind) { case "request": var request = MessagePackSerializer.Deserialize(vector.Bytes, CultNetSchemaMessageSerialization.Options); request.Should().BeEquivalentTo((CultMeshContentChunkRequestMessage)expected); Reencode(request).Should().Equal(vector.Bytes, vector.Label); break; case "response": var response = MessagePackSerializer.Deserialize(vector.Bytes, CultNetSchemaMessageSerialization.Options); response.Should().BeEquivalentTo((CultMeshContentChunkResponseMessage)expected); Reencode(response).Should().Equal(vector.Bytes, vector.Label); break; case "manifest": var manifest = ReadManifest(vector.Bytes); manifest.Should().BeEquivalentTo((CultMeshCdnArtifactManifest)expected); Reencode(manifest).Should().Equal(vector.Bytes, vector.Label); break; default: Assert.Fail($"unknown vector kind '{vector.Kind}'"); break; } } // Rust's answers to Rust's requests must be what the reference answers to the same bytes. foreach (var vector in rust.Where(v => v.Label.EndsWith(".request", StringComparison.Ordinal))) { var request = MessagePackSerializer.Deserialize(vector.Bytes, CultNetSchemaMessageSerialization.Options); var rustAnswer = rust.Single(v => v.Label == vector.Label[..^".request".Length] + ".response"); Reencode(Answer(cache, request)).Should().Equal(rustAnswer.Bytes, vector.Label); } } [Test] public void PackArtifactOfTheFixtureBodyEqualsTheFixtureManifest() { var artifact = CultMeshCdn.PackArtifact(ArtifactId, Body(), Options()); var committed = ReadVectors("content-vectors.cs-written.json").Single(v => v.Label == "manifest"); Reencode(artifact.Manifest).Should().Equal(committed.Bytes); var rust = ReadVectors("content-vectors.rs-written.json").Single(v => v.Label == "manifest"); rust.Bytes.Should().Equal(committed.Bytes, "Rust's pack_content and the reference pack the same bytes to the same manifest"); } [Test] public void CommittedRecordKeyVectorsEqualAFreshDerivation() { var path = VectorPath("cdn-record-key-vectors.json"); File.Exists(path).Should().BeTrue($"{path} is written by this fixture under CULTNET_WRITE_VECTORS=1"); Lines(File.ReadAllText(path)).Should().Be(Lines(RecordKeyVectorsToJson(BuildRecordKeyVectors())), "the key grammar drifted from the committed vectors"); } [Test] public void TypeScriptManifestVectorsMatchTheReference() { var path = VectorPath("content-vectors.ts-written.json"); File.Exists(path).Should().BeTrue($"{path} is written by scripts/write-cultmesh-content-vectors.mjs"); using var document = JsonDocument.Parse(File.ReadAllText(path)); var vectors = document.RootElement.GetProperty("vectors").EnumerateArray().ToList(); vectors.Should().NotBeEmpty(); foreach (var vector in vectors) { var label = vector.GetProperty("label").GetString()!; var bytes = Convert.FromBase64String(vector.GetProperty("messagePackBase64").GetString()!); var manifest = ReadManifest(bytes); Reencode(manifest).Should().Equal(bytes, $"{label}: the reference reads and rewrites the TypeScript bytes unchanged"); CultMeshCdnArtifactManifest.CreateRecordKey(manifest).Value.Should().Be(vector.GetProperty("recordKey").GetString(), label); var input = vector.GetProperty("input"); var body = input.GetProperty("body"); var multiplier = body.GetProperty("multiplier").GetInt64(); var modulus = body.GetProperty("modulus").GetInt64(); var packed = CultMeshCdn.PackArtifact( input.GetProperty("artifactId").GetString()!, Enumerable.Range(0, body.GetProperty("length").GetInt32()).Select(i => (byte)(i * multiplier % modulus)).ToArray(), new CultMeshCdnPackOptions { Kind = input.GetProperty("kind").GetString()!, Version = input.GetProperty("version").GetString()!, MimeType = input.GetProperty("mimeType").GetString()!, CreatedAtUtc = input.GetProperty("createdAtUtc").GetString()!, ChunkSizeBytes = input.GetProperty("chunkSizeBytes").GetInt32(), Tags = input.GetProperty("tags").EnumerateArray().Select(t => t.GetString()!).ToArray(), Metadata = input.GetProperty("metadata").EnumerateArray() .Aggregate(new Dictionary(), (map, pair) => { map[pair[0].GetString()!] = pair[1].GetString()!; return map; }) }); Reencode(packed.Manifest).Should().Equal(bytes, $"{label}: the reference packs the declared input to the TypeScript bytes"); } } [Test] public void CommittedAcceptanceVectorsEqualAFreshRun() { var path = VectorPath("content-acceptance-vectors.json"); File.Exists(path).Should().BeTrue($"{path} is written by this fixture under CULTNET_WRITE_VECTORS=1"); var fresh = BuildAcceptanceVectors(); fresh.Select(c => c.Label).Should().OnlyHaveUniqueItems(); fresh.Count(c => c.Accepted).Should().BeGreaterThan(30); fresh.Count(c => !c.Accepted).Should().BeGreaterThan(30); Lines(File.ReadAllText(path)).Should().Be(Lines(AcceptanceVectorsToJson(fresh)), "the reference's acceptance drifted from the committed vectors"); } private sealed record Acceptance(string Label, byte[] Input, bool Accepted, byte[]? Reencoded, string? RecordKey); /// What the reference does with these bytes: deserialise, then ValidateManifest; accepted bytes are rewritten and keyed. private static Acceptance Judge(string label, byte[] input) { CultMeshCdnArtifactManifest manifest; try { manifest = ReadManifest(input); CultMeshCdn.ValidateManifest(manifest); } catch (Exception) { return new Acceptance(label, input, false, null, null); } string? key = null; try { key = CultMeshCdnArtifactManifest.CreateRecordKey(manifest).Value; } catch (ArgumentException) { } return new Acceptance(label, input, true, Reencode(manifest), key); } private static byte[] Deep(int depth) => Cat(Enumerable.Repeat((byte)0x91, depth).ToArray(), Nil); private static byte[] Cat(params byte[][] parts) => parts.SelectMany(p => p).ToArray(); private static byte[] Raw(params byte[] bytes) => bytes; private static readonly byte[] Nil = { 0xc0 }; private static byte[] Str(string text) { var bytes = Encoding.UTF8.GetBytes(text); return Cat(bytes.Length < 32 ? Raw((byte)(0xa0 | bytes.Length)) : Raw(0xd9, (byte)bytes.Length), bytes); } private static byte[] StrRaw(params byte[] bytes) => Cat(Raw((byte)(0xa0 | bytes.Length)), bytes); private static byte[] Arr(params byte[][] items) => Cat(Raw((byte)(0x90 | items.Length)), Cat(items)); private static byte[] MapOf(params byte[][] pairs) => Cat(Raw((byte)(0x80 | (pairs.Length / 2))), Cat(pairs)); private static byte[] Be(byte code, long value, int width) { var bytes = new byte[width]; for (var i = 0; i < width; i++) bytes[width - 1 - i] = (byte)(value >> (8 * i)); return Cat(Raw(code), bytes); } private static byte[] F64(long bits) => Be(0xcb, bits, 8); private static byte[] F32(int bits) => Be(0xca, bits, 4); private static byte[][] BaseFields() => new[] { Str("a/b"), Str("asset"), Str("1"), Str(EmptyHash), Raw(0x00), Str("application/x-t"), Str("t"), Arr(Chunk(Raw(0x00), Raw(0x00))), Arr(Str("x")), MapOf(Str("k"), Str("v")) }; private const string EmptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; private static byte[] Chunk(byte[] offset, byte[] size) => Arr(Str(EmptyHash), offset, size, Str("k")); private static byte[] Doc(IEnumerable fields) { var list = fields.ToArray(); return Cat(Raw((byte)(0x90 | list.Length)), Cat(list)); } private static byte[][] With(params (int Index, byte[] Value)[] edits) { var fields = BaseFields(); foreach (var (index, value) in edits) fields[index] = value; return fields; } /// /// Byte shapes at the edge of what MessagePack-CSharp reads: nil in reference slots, missing and extra slots, /// trailing bytes, integer and float families, string families and invalid UTF-8. The reference judges each one; /// the TypeScript decoder is held to the verdict and to the bytes the reference writes back. /// private static List BuildAcceptanceVectors() { var cases = new List(); void Add(string label, byte[] input) => cases.Add(Judge(label, input)); void AddFields(string label, params (int Index, byte[] Value)[] edits) => Add(label, Doc(With(edits))); var fields = BaseFields(); Add("base", Doc(fields)); for (var count = 0; count <= 9; count++) Add($"short_{count}", Doc(fields.Take(count))); Add("array16_header", Cat(Raw(0xdc, 0, 10), Cat(fields))); Add("array32_header", Cat(Raw(0xdd, 0, 0, 0, 10), Cat(fields))); Add("extra_11_nil", Doc(fields.Append(Nil))); Add("extra_11_str", Doc(fields.Append(Str("x")))); Add("extra_11_nested", Doc(fields.Append(Arr(Arr(Str("x")), MapOf(Str("a"), F64(0)))))); Add("extra_11_ext", Doc(fields.Append(Raw(0xd4, 1, 2)))); Add("extra_11_ext8", Doc(fields.Append(Raw(0xc7, 2, 1, 7, 7)))); Add("extra_11_bin", Doc(fields.Append(Raw(0xc4, 2, 1, 2)))); Add("extra_11_float", Doc(fields.Append(F32(0)))); Add("extra_11_uint64", Doc(fields.Append(Be(0xcf, 7, 8)))); Add("extra_11_array16", Doc(fields.Append(Cat(Raw(0xdc, 0, 2), Raw(1), Raw(2))))); Add("extra_11_map16", Doc(fields.Append(Cat(Raw(0xde, 0, 1), Raw(1), Raw(2))))); Add("extra_11_str8", Doc(fields.Append(Raw(0xd9, 1, 0x41)))); Add("extra_12", Doc(fields.Append(Nil).Append(Str("y")))); Add("extra_11_reserved_code", Doc(fields.Append(Raw(0xc1)))); Add("extra_11_truncated", Doc(fields.Append(Raw(0x92, 1)))); // The production reader refuses payloads nested past CultPayloadDepth (256); the extra slot is skipped, so only that rule can refuse it. foreach (var depth in new[] { 250, 253, 254, 255, 256, 257, 300 }) { Add($"deep_extra_{depth}", Doc(fields.Append(Deep(depth)))); AddFields($"deep_chunkextra_{depth}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), Deep(depth))))); } AddFields("chunk_3", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0))))); // Slot 4 is the element id: a string, or nil (the reference reads null); anything else is refused. AddFields("chunk_5", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Nil)))); AddFields("chunk_5_nested", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Arr(Raw(1), Raw(2)))))); AddFields("chunk_id_str", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"))))); AddFields("chunk_id_int", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Raw(0))))); AddFields("chunk_id_str8", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Raw(0xd9, 1, 0x30))))); AddFields("chunk_6", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), Nil)))); // An extra slot (past the id) inside a chunk reference is followed by more bytes, so a skip that stops short shows. var extras = new (string Name, byte[] Value)[] { ("map", MapOf(Str("a"), Str("b"), Str("c"), Str("d"))), ("map16", Cat(Raw(0xde, 0, 2), Raw(1), Raw(2), Raw(3), Raw(4))), ("map32", Cat(Raw(0xdf, 0, 0, 0, 1), Raw(1), Raw(2))), ("array", Arr(Raw(1), Raw(2), Raw(3))), ("array16", Cat(Raw(0xdc, 0, 2), Raw(1), Raw(2))), ("array32", Cat(Raw(0xdd, 0, 0, 0, 1), Raw(1))), ("fixext1", Raw(0xd4, 1, 2)), ("fixext2", Raw(0xd5, 1, 2, 3)), ("fixext4", Raw(0xd6, 1, 2, 3, 4, 5)), ("fixext8", Raw(0xd7, 1, 2, 3, 4, 5, 6, 7, 8, 9)), ("fixext16", Cat(Raw(0xd8, 1), new byte[16])), ("ext8", Raw(0xc7, 2, 1, 7, 7)), ("ext16", Raw(0xc8, 0, 2, 1, 7, 7)), ("ext32", Raw(0xc9, 0, 0, 0, 2, 1, 7, 7)), ("bin8", Raw(0xc4, 2, 1, 2)), ("bin16", Raw(0xc5, 0, 2, 1, 2)), ("bin32", Raw(0xc6, 0, 0, 0, 2, 1, 2)), ("str8", Raw(0xd9, 2, 0x41, 0x42)), ("str16", Raw(0xda, 0, 2, 0x41, 0x42)), ("str32", Raw(0xdb, 0, 0, 0, 2, 0x41, 0x42)), ("fixstr", Str("ab")), ("f32", F32(1)), ("f64", F64(1)), ("u8", Raw(0xcc, 1)), ("u16", Be(0xcd, 1, 2)), ("u32", Be(0xce, 1, 4)), ("u64", Be(0xcf, 1, 8)), ("i8", Raw(0xd0, 1)), ("i16", Be(0xd1, 1, 2)), ("i32", Be(0xd2, 1, 4)), ("i64", Be(0xd3, 1, 8)), ("true", Raw(0xc3)), ("false", Raw(0xc2)), ("nil", Nil), ("negfixint", Raw(0xff)), ("reserved", Raw(0xc1)), ("fixint_max", Raw(0x7f)), ("negfixint_min", Raw(0xe0)), ("fixstr_31", Cat(Raw(0xbf), new byte[31])), ("fixmap_15", Cat(Raw(0x8f), Cat(Enumerable.Range(0, 15).Select(_ => Raw(1, 2)).ToArray()))), ("fixarray_15", Cat(Raw(0x9f), new byte[15])), ("nested", Arr(MapOf(Str("a"), Arr(Raw(1), Raw(2))), Arr())) }; foreach (var (name, value) in extras) AddFields($"chunk_extra_{name}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), value)))); AddFields("chunk_1", (7, Arr(Arr(Str(EmptyHash))))); AddFields("chunk_0", (7, Arr(Arr()))); AddFields("chunk_is_map", (7, Arr(MapOf(Str("a"), Str("b"))))); AddFields("chunk_is_str", (7, Arr(Str("a")))); AddFields("chunks_empty_with_size", (7, Arr())); AddFields("chunks_empty_size_zero", (7, Arr()), (4, Raw(0))); AddFields("chunks_is_str", (7, Str("a"))); AddFields("chunks_is_map", (7, MapOf())); AddFields("chunks_gap", (7, Arr(Chunk(Raw(1), Raw(0))))); foreach (var (index, name) in new[] { (0, "artifactId"), (1, "kind"), (2, "version"), (3, "contentHash"), (5, "mimeType"), (6, "createdAtUtc") }) AddFields($"nil_{name}", (index, Nil)); AddFields("nil_chunks", (7, Nil)); AddFields("nil_chunk_element", (7, Arr(Nil))); AddFields("nil_chunk_hash", (7, Arr(Arr(Nil, Raw(0), Raw(0), Str("k"))))); AddFields("nil_chunk_recordKey", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Nil)))); AddFields("nil_tags", (8, Nil)); AddFields("nil_tag_element", (8, Arr(Nil, Str("x")))); AddFields("nil_metadata", (9, Nil)); AddFields("nil_metadata_value", (9, MapOf(Str("k"), Nil))); AddFields("nil_metadata_key", (9, MapOf(Nil, Str("v")))); AddFields("nil_size", (4, Nil)); AddFields("metadata_duplicate_key", (9, MapOf(Str("k"), Str("1"), Str("k"), Str("2")))); AddFields("metadata_integer_like_keys", (9, MapOf(Str("10"), Str("a"), Str("alpha"), Str("b"), Str("2"), Str("c")))); AddFields("metadata_int_key", (9, MapOf(Raw(5), Str("v")))); AddFields("metadata_int_value", (9, MapOf(Str("k"), Raw(5)))); AddFields("metadata_empty_key", (9, MapOf(Str(""), Str("v")))); AddFields("metadata_proto_key", (9, MapOf(Str("__proto__"), Str("v")))); AddFields("metadata_map16", (9, Cat(Raw(0xde, 0, 1), Str("k"), Str("v")))); AddFields("metadata_map32", (9, Cat(Raw(0xdf, 0, 0, 0, 1), Str("k"), Str("v")))); AddFields("metadata_is_array", (9, Arr())); AddFields("metadata_is_str", (9, Str("k"))); AddFields("tags_is_map", (8, MapOf())); AddFields("tags_int_element", (8, Arr(Raw(5)))); AddFields("tags_array16", (8, Cat(Raw(0xdc, 0, 1), Str("x")))); Add("trailing_nil", Cat(Doc(fields), Nil)); Add("trailing_reserved_code", Cat(Doc(fields), Raw(0xc1))); Add("trailing_truncated_array", Cat(Doc(fields), Raw(0x92))); Add("top_nil", Nil); Add("top_empty", Array.Empty()); Add("top_map", MapOf(Str("a"), Str("b"))); Add("top_str", Str("a")); Add("top_int", Raw(5)); Add("top_truncated", Doc(fields)[..^3]); foreach (var (name, zero) in new (string, byte[])[] { ("u8", Raw(0xcc, 0)), ("u16", Be(0xcd, 0, 2)), ("u32", Be(0xce, 0, 4)), ("u64", Be(0xcf, 0, 8)), ("i8", Raw(0xd0, 0)), ("i16", Be(0xd1, 0, 2)), ("i32", Be(0xd2, 0, 4)), ("i64", Be(0xd3, 0, 8)), ("f32", F32(0)), ("f32_negzero", F32(int.MinValue)), ("f64", F64(0)), ("f64_negzero", F64(long.MinValue)), ("f64_one", F64(0x3FF0000000000000)), ("true", Raw(0xc3)), ("false", Raw(0xc2)), ("str", Str("0")), ("bin", Raw(0xc4, 1, 0)), ("array", Arr()), ("ext", Raw(0xd4, 1, 0)), // Zero-filled so that a reader which took them for an int32 would read 0. ("fixext4", Cat(Raw(0xd6), new byte[5])), ("fixext8", Cat(Raw(0xd7), new byte[9])), ("fixext16", Cat(Raw(0xd8), new byte[17])) }) { AddFields($"size_{name}", (4, zero)); AddFields($"offset_{name}", (7, Arr(Chunk(zero, Raw(0))))); AddFields($"chunksize_{name}", (7, Arr(Chunk(Raw(0), zero)))); } AddFields("size_negative", (4, Raw(0xff)), (7, Arr())); AddFields("size_u64_over_long", (4, Be(0xcf, -1, 8)), (7, Arr())); AddFields("size_i64_min", (4, Be(0xd3, long.MinValue, 8)), (7, Arr())); AddFields("size_2p53_minus_1", (4, Be(0xcf, (1L << 53) - 1, 8)), (7, Arr())); AddFields("chunksize_int_max", (4, Be(0xce, int.MaxValue, 4)), (7, Arr(Chunk(Raw(0), Be(0xce, int.MaxValue, 4))))); AddFields("chunksize_2p31", (4, Be(0xce, 1L << 31, 4)), (7, Arr(Chunk(Raw(0), Be(0xce, 1L << 31, 4))))); AddFields("chunksize_i32_min", (4, Raw(0)), (7, Arr(Chunk(Raw(0), Be(0xd2, int.MinValue, 4))))); AddFields("chunksize_i64_below_i32", (4, Raw(0)), (7, Arr(Chunk(Raw(0), Be(0xd3, (long)int.MinValue - 1, 8))))); AddFields("offset_beyond_int32", (4, Be(0xcf, 3L * int.MaxValue, 8)), (7, Arr(Chunk(Raw(0), Be(0xce, int.MaxValue, 4)), Chunk(Be(0xce, int.MaxValue, 4), Be(0xce, int.MaxValue, 4)), Chunk(Be(0xce, 2L * int.MaxValue, 4), Be(0xce, int.MaxValue, 4))))); AddFields("chunks_out_of_order", (4, Raw(2)), (7, Arr(Chunk(Raw(1), Raw(1)), Chunk(Raw(0), Raw(1))))); AddFields("chunks_overlap", (4, Raw(2)), (7, Arr(Chunk(Raw(0), Raw(2)), Chunk(Raw(1), Raw(1))))); AddFields("chunk_negative_size", (4, Raw(0)), (7, Arr(Chunk(Raw(0), Raw(0xff))))); AddFields("str_bin", (1, Raw(0xc4, 1, 0x41))); AddFields("str_ext", (1, Raw(0xd4, 1, 0x41))); AddFields("str_int", (1, Raw(0x41))); AddFields("str_str16", (1, Raw(0xda, 0, 1, 0x41))); AddFields("str_str32", (1, Raw(0xdb, 0, 0, 0, 1, 0x41))); AddFields("str_empty", (1, Str(""))); AddFields("str_invalid_utf8", (1, StrRaw(0xFF, 0xFE))); AddFields("str_lone_surrogate_cesu", (1, StrRaw(0xED, 0xA0, 0x80))); AddFields("str_overlong", (1, StrRaw(0xC0, 0xAF))); AddFields("str_truncated_sequence", (1, StrRaw(0xE2, 0x82))); AddFields("str_bom", (0, Str("bom"))); AddFields("str_nul", (0, StrRaw(0x61, 0x00, 0x62))); AddFields("id_invalid_utf8", (0, StrRaw(0xFF))); AddFields("hash_invalid_utf8", (3, StrRaw(0xFF))); AddFields("version_invalid_utf8", (2, StrRaw(0xFF, 0xFE))); AddFields("id_blank", (0, Str(" "))); AddFields("hash_blank", (3, Str(" "))); AddFields("hash_prefix_upper_long_s", (3, Str("ſha256:abc"))); // Shapes that end early, and header families a lenient reader would take for another. var whole = Doc(fields); Add("truncated_string_body", whole[..^1]); Add("truncated_str8_length", Raw(0x91, 0xd9)); Add("truncated_u16", Cat(Raw(0x95), Str("a"), Str("k"), Str("v"), Str(EmptyHash), Raw(0xcd, 0))); Add("truncated_u64", Cat(Raw(0x95), Str("a"), Str("k"), Str("v"), Str(EmptyHash), Raw(0xcf, 0, 0, 0))); Add("truncated_array16", Raw(0xdc, 0)); Add("extra_11_truncated_string", Doc(fields.Append(Raw(0xa5, 0x41)))); Add("extra_11_truncated_str8_length", Doc(fields.Append(Raw(0xd9)))); Add("array_15", Doc(fields.Concat(Enumerable.Repeat(Nil, 5)))); AddFields("metadata_15_entries", (9, Cat(Raw(0x8f), Cat(Enumerable.Range(0, 15).Select(i => Cat(Str($"k{i}"), Str("v"))).ToArray())))); Add("tags_fixint_then_trailing", Cat(Raw(0x99), Cat(fields[..8]), Raw(1), Str("a"))); Add("tags_fixmap", Cat(Raw(0x99), Cat(fields[..8]), MapOf(Str("a"), Str("b")))); Add("metadata_fixint_then_trailing", Cat(Raw(0x9a), Cat(fields[..9]), Raw(1), Str("a"), Str("b"))); Add("metadata_array_then_trailing", Cat(Raw(0x9a), Cat(fields[..9]), Arr(Str("a"), Str("b")), Str("c"), Str("d"))); // Every integer family at a value that needs its full width and its own signedness: as the sizes, then as an offset. foreach (var (name, value, n) in new (string, byte[], long)[] { ("u8_200", Raw(0xcc, 200), 200), ("u16_258", Be(0xcd, 258, 2), 258), ("u16_40000", Be(0xcd, 40000, 2), 40000), ("u32_258", Be(0xce, 258, 4), 258), ("u64_258", Be(0xcf, 258, 8), 258), ("i8_100", Raw(0xd0, 100), 100), ("i16_258", Be(0xd1, 258, 2), 258), ("i32_258", Be(0xd2, 258, 4), 258), ("i64_258", Be(0xd3, 258, 8), 258), ("fixint_127", Raw(0x7f), 127) }) { AddFields($"width_{name}", (4, value), (7, Arr(Chunk(Raw(0), value)))); AddFields($"width_{name}_offset", (4, Be(0xcd, n + 1, 2)), (7, Arr(Chunk(Raw(0), Be(0xcd, n, 2)), Chunk(value, Raw(1))))); } AddFields("width_u32_over_int32", (4, Be(0xce, 3_000_000_000, 4)), (7, Arr(Chunk(Raw(0), Be(0xce, int.MaxValue, 4)), Chunk(Be(0xce, int.MaxValue, 4), Be(0xce, 3_000_000_000 - int.MaxValue, 4))))); AddFields("width_negative_fixint", (4, Raw(0xff))); AddFields("width_negative_fixint_min", (4, Raw(0xe0))); AddFields("width_negative_i64_safe", (4, Be(0xd3, -((1L << 53) - 1), 8))); AddFields("width_negative_i64_unsafe", (4, Be(0xd3, -(1L << 53), 8))); return cases; } private static string AcceptanceVectorsToJson(List cases) => JsonSerializer.Serialize( new { cases = cases.Select(c => new { label = c.Label, inputBase64 = Convert.ToBase64String(c.Input), accepted = c.Accepted, reencodedBase64 = c.Reencoded == null ? null : Convert.ToBase64String(c.Reencoded), recordKey = c.RecordKey }) }, new JsonSerializerOptions { WriteIndented = true }); private static byte[] Body() => Enumerable.Range(0, BodyLength).Select(i => (byte)(i % 251)).ToArray(); private static CultMeshCdnPackOptions Options() => new() { ChunkSizeBytes = ChunkSize, Version = "1", MimeType = "application/x-cultnet-vector", CreatedAtUtc = CreatedAtUtc }; private static byte[] Reencode(CultMeshContentChunkRequestMessage message) => CultNetSchemaMessageSerialization.Serialize(message); private static byte[] Reencode(CultMeshContentChunkResponseMessage message) => CultNetSchemaMessageSerialization.Serialize(message); /// The one reader of manifest bytes: the document reader production stores and reads manifests with (CultPayloadDepth 256 included). private static CultMeshCdnArtifactManifest ReadManifest(byte[] bytes) => (CultMeshCdnArtifactManifest)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultMeshCdnArtifactManifest), bytes); private static byte[] Reencode(CultMeshCdnArtifactManifest manifest) => CultDocumentMessagePackSerialization.SerializeUntyped(manifest, typeof(CultMeshCdnArtifactManifest)); private static CultMeshContentChunkRequestMessage Request(string id, string hash, string key, int size) => new() { MessageId = id, ChunkHash = hash, RecordKey = key, ExpectedSizeBytes = size }; /// The cs-written file's vectors, in file order. Messages are keyed by label. private static List BuildVectors(out CultCache cache, out Dictionary messages) { var artifact = CultMeshCdn.PackArtifact(ArtifactId, Body(), Options()); var content = new CultCache(CultMesh.CreateCultCacheDocumentRegistry( typeof(CultMeshCdnArtifactManifest), typeof(CultMeshCdnArtifactChunk))); CultMeshCdn.PublishAsync(content, artifact).GetAwaiter().GetResult(); cache = content; var last = artifact.Chunks[^1]; var key = "mesh:cdn:chunk:" + last.ChunkHash; var lastDigit = last.ChunkHash[^1] == '0' ? '1' : '0'; var vectors = new List(); var map = new Dictionary(StringComparer.Ordinal); messages = map; void AddRequest(string label, CultMeshContentChunkRequestMessage message) { map[label] = message; vectors.Add(new Vector(label, "request", Reencode(message))); } void AddResponse(string label, CultMeshContentChunkResponseMessage message) { map[label] = message; vectors.Add(new Vector(label, "response", Reencode(message))); } void AddAnswer(string name, CultMeshContentChunkRequestMessage request) { AddRequest(name + ".request", request); AddResponse(name + ".response", Answer(content, request)); } AddAnswer("answer_found", Request("vector-found", "SHA256:" + last.ChunkHash, key, last.SizeBytes)); AddAnswer("answer_not_found", Request("vector-missing", new string('0', 64), string.Empty, 100)); AddAnswer("answer_record_key_disagrees", Request("vector-key", last.ChunkHash, "mesh:cdn:chunk:other", last.SizeBytes)); AddAnswer("answer_size_mismatch", Request("vector-size", last.ChunkHash, key, last.SizeBytes - 1)); // Requests the reference refuses in its handler, never at deserialisation: each is answered. AddAnswer("answer_blank_message_id", Request(string.Empty, last.ChunkHash, key, last.SizeBytes)); AddAnswer("answer_whitespace_message_id", Request(" ", last.ChunkHash, key, last.SizeBytes)); AddAnswer("answer_blank_hash", Request("vector-blank-hash", string.Empty, string.Empty, 100)); AddAnswer("answer_whitespace_hash", Request("vector-space-hash", " ", string.Empty, 100)); AddAnswer("answer_prefix_only_hash", Request("vector-prefix-hash", "sha256: ", string.Empty, 100)); AddAnswer("answer_negative_size", Request("vector-negative", last.ChunkHash, key, -1)); AddAnswer("answer_record_key_case", Request("vector-key-case", last.ChunkHash, key.ToUpperInvariant(), last.SizeBytes)); AddAnswer("answer_blank_record_key", Request("vector-key-blank", last.ChunkHash, " ", last.SizeBytes)); AddAnswer("answer_double_prefix", Request("vector-double", "sha256:sha256:" + last.ChunkHash, string.Empty, last.SizeBytes)); // A caller that leaves a string null writes msgpack nil; the reference reads it back as null and // answers. These are judged by Rust decoding the reference's bytes (Rust never writes nil), so // they stay out of the label-matched mirror set. void AddNilAnswer(string name, CultMeshContentChunkRequestMessage request) { vectors.Add(new Vector(name + ".request", "request", Reencode(request))); vectors.Add(new Vector(name + ".response", "response", Reencode(Answer(content, request)))); } AddNilAnswer("nil_message_id", Request(null, last.ChunkHash, key, last.SizeBytes)); AddNilAnswer("nil_hash", Request("vector-nil-hash", null, string.Empty, 100)); AddNilAnswer("nil_record_key", Request("vector-nil-key", new string('0', 64), null, 100)); AddResponse("response_hash_last_digit", new CultMeshContentChunkResponseMessage { MessageId = "vector-digit", Found = true, ChunkHash = last.ChunkHash[..^1] + lastDigit, SizeBytes = last.SizeBytes, Payload = last.Payload, Error = string.Empty }); void AddManifest(string label, CultMeshCdnArtifactManifest manifest) { map[label] = manifest; vectors.Add(new Vector(label, "manifest", Reencode(manifest))); } AddManifest("manifest", artifact.Manifest); var shuffled = CultMeshCdn.PackArtifact(ArtifactId, Body(), Options()).Manifest; shuffled.Chunks = new[] { 3, 0, 5, 1, 4, 2 }.Select(i => shuffled.Chunks[i]).ToArray(); AddManifest("manifest_out_of_order", shuffled); var tagged = CultMeshCdn.PackArtifact(ArtifactId, Body(), Options()).Manifest; tagged.Metadata = new Dictionary { ["zeta"] = "1", ["alpha"] = "2", ["mid"] = "3" }; AddManifest("manifest_metadata_order", tagged); AddManifest("manifest_empty_body", CultMeshCdn.PackArtifact(ArtifactId, Array.Empty(), Options()).Manifest); return vectors; } private static CultMeshContentChunkResponseMessage Answer(CultCache cache, CultMeshContentChunkRequestMessage request) { var host = new ReferenceHost(); using var server = new CultMeshLegacyRudpContentServer(host, cache); var peer = new ReferencePeer(); host.Dispatch(request, peer).GetAwaiter().GetResult(); return peer.Sent ?? throw new InvalidOperationException("The reference server answered with silence."); } private sealed class ReferenceHost : ICultNetSchemaServer { private Func? _handler; public void OnCultNet(Func callback) where TMessage : ICultNetSchemaMessage => _handler = (Func)(object)callback; public void RemoveCultNetMessageListener(Delegate callback) where TMessage : ICultNetSchemaMessage => _handler = null; public Task Dispatch(CultMeshContentChunkRequestMessage request, ICultNetSchemaServerPeer peer) => _handler!(request, peer); } private sealed class ReferencePeer : ICultNetSchemaServerPeer { public CultMeshContentChunkResponseMessage? Sent { get; private set; } public void SendCultNet(TMessage message) where TMessage : ICultNetSchemaMessage => Sent = (CultMeshContentChunkResponseMessage)(object)message; } private static string ToJson(List vectors) => JsonSerializer.Serialize( new { vectors = vectors.Select(v => new { label = v.Label, kind = v.Kind, messagePackBase64 = Convert.ToBase64String(v.Bytes) }) }, new JsonSerializerOptions { WriteIndented = true }); private static List ReadVectors(string name) { var path = VectorPath(name); if (!File.Exists(path)) Assert.Inconclusive($"{path} is missing - write it first (CULTNET_WRITE_VECTORS=1)."); using var document = JsonDocument.Parse(File.ReadAllText(path)); return document.RootElement.GetProperty("vectors").EnumerateArray() .Select(v => new Vector( v.GetProperty("label").GetString()!, v.GetProperty("kind").GetString()!, Convert.FromBase64String(v.GetProperty("messagePackBase64").GetString()!))) .ToList(); } private static string Lines(string text) => text.Replace("\r\n", "\n").TrimEnd('\n'); private static string[] KeyHashes() { var empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; return new[] { empty, "SHA256:" + empty.ToUpperInvariant(), " " + empty + " ", " sha256:" + empty, "sha256:sha256:" + empty, "Sha256:\u0130\u03A3\U00010400", "sha256:", // U+017F uppercases to 'S': a prefix fold by upper case would strip it, the reference's ordinal-ignore-case does not. "\u017Fha256:abc" }; } private sealed record KeyVectors( List<(string Id, string Version, string Hash, string Key)> Artifacts, List<(string Hash, string Key)> Chunks); /// Every key is derived by the reference's public CreateRecordKey, never spelled by hand. private static KeyVectors BuildRecordKeyVectors() { var ids = new[] { "eureka/content-vector", " Mixed Case/Path ", "\u0130stanbul", "\u0391\u03A3", "\uFEFFbom", "\u0085nel\u0085", "\u65E5\u672C\u8A9E/\u8CC7\u7523", "\U0001F642 smile", "---", "a\uD800b", "\uD800", "x\u00B2y", "\u0663d", "\U00010400A", // Tokens that are empty after the trim: the fallback hashes the original, neither trimmed nor lower-cased. " --- ", "\u24B6", "\u216B" }; var versions = new[] { string.Empty, " ", "1.0.0", "V2 Beta", "\u00FC" }; var hashes = KeyHashes(); var artifacts = ids.Select((id, i) => (id, versions[i % versions.Length], hashes[i % hashes.Length])) .Concat(versions.Select(version => (ids[0], version, hashes[0]))) .Concat(hashes.Select(hash => (ids[0], versions[0], hash))) .Select(t => (t.Item1, t.Item2, t.Item3, CultMeshCdnArtifactManifest.CreateRecordKey(t.Item1, t.Item2, t.Item3).Value)) .ToList(); var chunks = hashes.Select(hash => (hash, CultMeshCdnArtifactChunk.CreateRecordKey(hash).Value)).ToList(); return new KeyVectors(artifacts, chunks); } /// String inputs are stored as UTF-16 code-unit arrays, so a lone surrogate survives the JSON file. private static string RecordKeyVectorsToJson(KeyVectors vectors) => JsonSerializer.Serialize( new { artifacts = vectors.Artifacts.Select(a => new { artifactIdUnits = a.Id.Select(c => (int)c), versionUnits = a.Version.Select(c => (int)c), contentHash = a.Hash, recordKey = a.Key }), chunks = vectors.Chunks.Select(c => new { chunkHash = c.Hash, recordKey = c.Key }) }, new JsonSerializerOptions { WriteIndented = true }); private static string VectorPath(string name) => Path.Combine(RepoRoot(), "contracts", "cultmesh", name); private static string RepoRoot() { var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) directory = directory.Parent; return directory?.FullName ?? throw new InvalidOperationException("CultLib.sln was not found above the test directory."); } }