From 139deb0e28b3764339e16a796f2b8ddea4ddb0be Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:13:22 +0200 Subject: [PATCH 01/64] Add the canonical MessagePack encoding for every public CultMath value type CultMathResolver formats float2/3/4, double2/3, int2/3/4, bool2/3/4, quaternion, Color32, Random, float2x2, float3x3, rect and CultCellular as component arrays, and sits in the default CultCache resolver chain so no consumer registers anything. It lives in GameCult.Caching.MessagePack, which already owns that chain; CultMath core stays dependency-free. Vector, rect and Aetheria-formatter byte shapes are pinned by golden tests, a reflection test fails when a public CultMath value type has no formatter, and decode skips extra elements and refuses nil. Co-Authored-By: Claude Sonnet 5 --- .../CultDocumentMessagePackSerialization.cs | 2 +- .../CultMathResolver.cs | 231 ++++++++++++++++++ .../CultMathSerializationTests.cs | 149 +++++++++++ 3 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 src/GameCult.Caching.MessagePack/CultMathResolver.cs create mode 100644 tests/GameCult.Caching.Tests/CultMathSerializationTests.cs diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index ef1b4092..f591d764 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -57,7 +57,7 @@ public static class CultDocumentMessagePackSerialization { return MessagePackSerializerOptions.Standard .WithResolver(CompositeResolver.Create( - consumerResolvers.Append(CultDocumentResolver.Instance).Append(StandardResolver.Instance).ToArray())) + consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(StandardResolver.Instance).ToArray())) .WithSecurity(CultMessagePackSecurity.Instance); } diff --git a/src/GameCult.Caching.MessagePack/CultMathResolver.cs b/src/GameCult.Caching.MessagePack/CultMathResolver.cs new file mode 100644 index 00000000..71bdec2b --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultMathResolver.cs @@ -0,0 +1,231 @@ +using System; +using System.Collections.Generic; +using CultMath; +using MessagePack; +using MessagePack.Formatters; +using Random = CultMath.Random; + +namespace GameCult.Caching.MessagePack; + +/// +/// The canonical MessagePack encoding of every public CultMath value type. Each type is an +/// array of its components in declaration order: vectors and quaternions are flat scalar +/// arrays, matrices are arrays of row vectors, and rect is [min, max]. Decoding +/// skips trailing elements and leaves missing ones at zero; nil is refused. The resolver is +/// part of the default chain, so a consumer +/// registers nothing. +/// +public sealed class CultMathResolver : IFormatterResolver +{ + public static readonly CultMathResolver Instance = new(); + private CultMathResolver() { } + + /// The CultMath types this resolver formats, for coverage checks. + public static IReadOnlyCollection FormattedTypes => Formatters.Keys; + + public IMessagePackFormatter? GetFormatter() => Cache.Formatter; + + private static class Cache + { + public static readonly IMessagePackFormatter? Formatter = + Formatters.TryGetValue(typeof(T), out var formatter) ? (IMessagePackFormatter)formatter : null; + } + + private delegate void WriteFn(ref MessagePackWriter writer, T value); + private delegate T ReadFn(ref MessagePackReader reader); + + private sealed class ArrayShape : IMessagePackFormatter + { + private readonly WriteFn write; + private readonly ReadFn read; + + public ArrayShape(WriteFn write, ReadFn read) + { + this.write = write; + this.read = read; + } + + public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options) => write(ref writer, value); + + public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + if (reader.IsNil) throw new MessagePackSerializationException($"{typeof(T).Name} cannot be read from nil."); + return read(ref reader); + } + } + + private static KeyValuePair Shape(WriteFn write, ReadFn read) => + new(typeof(T), new ArrayShape(write, read)); + + // Each reader takes the first dst.Length elements and skips the rest. + private static void Singles(ref MessagePackReader r, Span dst) + { + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < dst.Length) dst[i] = r.ReadSingle(); else r.Skip(); + } + + private static void Doubles(ref MessagePackReader r, Span dst) + { + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < dst.Length) dst[i] = r.ReadDouble(); else r.Skip(); + } + + private static void Ints(ref MessagePackReader r, Span dst) + { + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < dst.Length) dst[i] = r.ReadInt32(); else r.Skip(); + } + + private static void Bools(ref MessagePackReader r, Span dst) + { + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < dst.Length) dst[i] = r.ReadBoolean(); else r.Skip(); + } + + private static void Bytes(ref MessagePackReader r, Span dst) + { + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < dst.Length) dst[i] = r.ReadByte(); else r.Skip(); + } + + private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + { + w.WriteArrayHeader(v.Length); + foreach (var e in v) w.Write(e); + } + + private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + { + w.WriteArrayHeader(v.Length); + foreach (var e in v) w.Write(e); + } + + private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + { + w.WriteArrayHeader(v.Length); + foreach (var e in v) w.Write(e); + } + + private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + { + w.WriteArrayHeader(v.Length); + foreach (var e in v) w.Write(e); + } + + private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + { + w.WriteArrayHeader(v.Length); + foreach (var e in v) w.Write(e); + } + + private static void PutFloat2(ref MessagePackWriter w, float2 v) => Put(ref w, stackalloc float[] { v.x, v.y }); + private static void PutFloat3(ref MessagePackWriter w, float3 v) => Put(ref w, stackalloc float[] { v.x, v.y, v.z }); + private static void PutFloat4(ref MessagePackWriter w, float4 v) => Put(ref w, stackalloc float[] { v.x, v.y, v.z, v.w }); + + private static float2 GetFloat2(ref MessagePackReader r) + { + Span s = stackalloc float[2]; + Singles(ref r, s); + return new float2(s[0], s[1]); + } + + private static float3 GetFloat3(ref MessagePackReader r) + { + Span s = stackalloc float[3]; + Singles(ref r, s); + return new float3(s[0], s[1], s[2]); + } + + private static float4 GetFloat4(ref MessagePackReader r) + { + Span s = stackalloc float[4]; + Singles(ref r, s); + return new float4(s[0], s[1], s[2], s[3]); + } + + private static readonly Dictionary Formatters = Build(); + + private static Dictionary Build() + { + var shapes = new[] + { + Shape(PutFloat2, GetFloat2), + Shape(PutFloat3, GetFloat3), + Shape(PutFloat4, GetFloat4), + Shape((ref MessagePackWriter w, double2 v) => Put(ref w, stackalloc double[] { v.x, v.y }), + (ref MessagePackReader r) => { Span s = stackalloc double[2]; Doubles(ref r, s); return new double2(s[0], s[1]); }), + Shape((ref MessagePackWriter w, double3 v) => Put(ref w, stackalloc double[] { v.x, v.y, v.z }), + (ref MessagePackReader r) => { Span s = stackalloc double[3]; Doubles(ref r, s); return new double3(s[0], s[1], s[2]); }), + Shape((ref MessagePackWriter w, int2 v) => Put(ref w, stackalloc int[] { v.x, v.y }), + (ref MessagePackReader r) => { Span s = stackalloc int[2]; Ints(ref r, s); return new int2(s[0], s[1]); }), + Shape((ref MessagePackWriter w, int3 v) => Put(ref w, stackalloc int[] { v.x, v.y, v.z }), + (ref MessagePackReader r) => { Span s = stackalloc int[3]; Ints(ref r, s); return new int3(s[0], s[1], s[2]); }), + Shape((ref MessagePackWriter w, int4 v) => Put(ref w, stackalloc int[] { v.x, v.y, v.z, v.w }), + (ref MessagePackReader r) => { Span s = stackalloc int[4]; Ints(ref r, s); return new int4(s[0], s[1], s[2], s[3]); }), + Shape((ref MessagePackWriter w, bool2 v) => Put(ref w, stackalloc bool[] { v.x, v.y }), + (ref MessagePackReader r) => { Span s = stackalloc bool[2]; Bools(ref r, s); return new bool2(s[0], s[1]); }), + Shape((ref MessagePackWriter w, bool3 v) => Put(ref w, stackalloc bool[] { v.x, v.y, v.z }), + (ref MessagePackReader r) => { Span s = stackalloc bool[3]; Bools(ref r, s); return new bool3(s[0], s[1], s[2]); }), + Shape((ref MessagePackWriter w, bool4 v) => Put(ref w, stackalloc bool[] { v.x, v.y, v.z, v.w }), + (ref MessagePackReader r) => { Span s = stackalloc bool[4]; Bools(ref r, s); return new bool4(s[0], s[1], s[2], s[3]); }), + Shape((ref MessagePackWriter w, quaternion v) => Put(ref w, stackalloc float[] { v.x, v.y, v.z, v.w }), + (ref MessagePackReader r) => { Span s = stackalloc float[4]; Singles(ref r, s); return new quaternion(s[0], s[1], s[2], s[3]); }), + Shape((ref MessagePackWriter w, Color32 v) => Put(ref w, stackalloc byte[] { v.r, v.g, v.b, v.a }), + (ref MessagePackReader r) => { Span s = stackalloc byte[4]; Bytes(ref r, s); return new Color32(s[0], s[1], s[2], s[3]); }), + Shape((ref MessagePackWriter w, Random v) => { w.WriteArrayHeader(1); w.Write(v.state); }, + (ref MessagePackReader r) => + { + var value = default(Random); + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i == 0) value.state = r.ReadUInt32(); else r.Skip(); + return value; + }), + + // Compositions: matrices are rows, rect is [min, max], CultCellular is [nearest, edge, id]. + Shape((ref MessagePackWriter w, float2x2 v) => { w.WriteArrayHeader(2); PutFloat2(ref w, v[0]); PutFloat2(ref w, v[1]); }, + (ref MessagePackReader r) => + { + var rows = new float2[2]; + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < 2) rows[i] = GetFloat2(ref r); else r.Skip(); + return new float2x2(rows[0], rows[1]); + }), + Shape((ref MessagePackWriter w, float3x3 v) => { w.WriteArrayHeader(3); PutFloat3(ref w, v[0]); PutFloat3(ref w, v[1]); PutFloat3(ref w, v[2]); }, + (ref MessagePackReader r) => + { + var rows = new float3[3]; + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < 3) rows[i] = GetFloat3(ref r); else r.Skip(); + return new float3x3(rows[0], rows[1], rows[2]); + }), + Shape((ref MessagePackWriter w, rect v) => { w.WriteArrayHeader(2); PutFloat2(ref w, v.min); PutFloat2(ref w, v.max); }, + (ref MessagePackReader r) => + { + var corners = new float2[2]; + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + if (i < 2) corners[i] = GetFloat2(ref r); else r.Skip(); + return new rect(corners[0], corners[1]); + }), + Shape((ref MessagePackWriter w, CultCellular v) => { w.WriteArrayHeader(3); PutFloat4(ref w, v.nearest); PutFloat4(ref w, v.edge); w.Write(v.id); }, + (ref MessagePackReader r) => + { + var value = default(CultCellular); + for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) + { + switch (i) + { + case 0: value.nearest = GetFloat4(ref r); break; + case 1: value.edge = GetFloat4(ref r); break; + case 2: value.id = r.ReadSingle(); break; + default: r.Skip(); break; + } + } + return value; + }), + }; + + var map = new Dictionary(); + foreach (var shape in shapes) map.Add(shape.Key, shape.Value); + return map; + } +} diff --git a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs new file mode 100644 index 00000000..c6b6d161 --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using GameCult.Caching.MessagePack; +using CultMath; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + public sealed class CultMathSerializationTests + { + private static readonly MessagePackSerializerOptions Options = CultDocumentMessagePackSerialization.Options; + + private static string Hex(T value) => Convert.ToHexString(MessagePackSerializer.Serialize(value, Options)); + + private static T Decode(string hex) => MessagePackSerializer.Deserialize(Convert.FromHexString(hex), Options); + + [MessagePackObject] + public sealed class Carrier + { + [Key(0)] public float2 Position; + [Key(1)] public float3? Velocity; + [Key(2)] public float2[] Path = Array.Empty(); + [Key(3)] public List Cells = new(); + } + + // Bytes the pre-collapse CultVec2/CultVec3/CultRect encodings and Aetheria's Float2/Int2/Bool2/Float3/Float4 + // formatters produce: an array of components in declaration order, float32 as 0xCA. + [TestCase("92CA3F800000CA40000000")] + public void Float2_MatchesTheLegacyVectorBytes(string hex) + { + Hex(new float2(1f, 2f)).Should().Be(hex); + Decode(hex).Should().Be(new float2(1f, 2f)); + } + + [Test] + public void Float3_MatchesTheLegacyVectorBytes() + { + const string hex = "93CA3F800000CA40000000CA40400000"; + Hex(new float3(1f, 2f, 3f)).Should().Be(hex); + Decode(hex).Should().Be(new float3(1f, 2f, 3f)); + } + + [Test] + public void Rect_MatchesTheLegacyCultRectBytes() + { + const string hex = "9292CABF800000CAC000000092CA40A00000CA40E00000"; + Hex(new rect(5f, 7f, -1f, -2f)).Should().Be(hex); + Decode(hex).Should().Be(new rect(-1f, -2f, 5f, 7f)); + } + + [Test] + public void ConsumerFormatterShapes_MatchByteForByte() + { + Hex(new float4(1f, 2f, 3f, 4f)).Should().Be("94CA3F800000CA40000000CA40400000CA40800000"); + Hex(new int2(3, -4)).Should().Be("9203FC"); + Hex(new bool2(true, false)).Should().Be("92C3C2"); + } + + [Test] + public void Decode_SkipsExtraElements_AndLeavesMissingOnesZero() + { + Decode("93CA3F800000CA40000000CA40400000").Should().Be(new float2(1f, 2f)); + Decode("92CA3F800000CA40000000").Should().Be(new float3(1f, 2f, 0f)); + Decode("9392CABF800000CAC000000092CA40A00000CA40E0000001").Should().Be(new rect(-1f, -2f, 5f, 7f)); + } + + [Test] + public void Decode_RefusesNilAndNonArrays() + { + Action nil = () => Decode("C0"); + nil.Should().Throw(); + Action notArray = () => Decode("01"); + notArray.Should().Throw(); + } + + [Test] + public void EveryFormattedType_RoundTrips() + { + RoundTrip(new float2(1.5f, -2f)); + RoundTrip(new float3(1f, 2f, 3f)); + RoundTrip(new float4(1f, 2f, 3f, 4f)); + RoundTrip(new double2(1e300, -2.5)); + RoundTrip(new double3(1, 2, 3)); + RoundTrip(new int2(-7, 900000)); + RoundTrip(new int3(1, -2, 3)); + RoundTrip(new int4(1, 2, 3, int.MinValue)); + RoundTrip(new bool2(true, false)); + RoundTrip(new bool3(false, true, true)); + RoundTrip(new bool4(true, false, false, true)); + RoundTrip(new quaternion(0.1f, 0.2f, 0.3f, 0.9f)); + RoundTrip(new Color32(255, 128, 0, 7)); + RoundTrip(new CultMath.Random(12345u)); + RoundTrip(new float2x2(1f, 2f, 3f, 4f)); + RoundTrip(new float3x3(1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f)); + RoundTrip(new rect(-1f, -2f, 5f, 7f)); + RoundTrip(new CultCellular(new float4(1, 2, 3, 4), new float4(5, 6, 7, 8), 0.25f)); + } + + [Test] + public void Matrices_AreArraysOfRows() + { + Hex(new float2x2(1f, 2f, 3f, 4f)).Should().Be("9292CA3F800000CA400000009 2CA40400000CA40800000".Replace(" ", "")); + } + + [Test] + public void EveryPublicCultMathValueType_HasAFormatter() + { + var valueTypes = typeof(float2).Assembly.GetExportedTypes() + .Where(t => t.IsValueType && !t.IsEnum) + .ToHashSet(); + + valueTypes.Except(CultMathResolver.FormattedTypes).Should().BeEmpty( + "every public CultMath value type serializes through CultMathResolver"); + CultMathResolver.FormattedTypes.Except(valueTypes).Should().BeEmpty(); + } + + [Test] + public void TheDefaultChain_ComposesMathIntoNullablesCollectionsAndDocuments() + { + var value = new Carrier + { + Position = new float2(1f, 2f), + Velocity = new float3(3f, 4f, 5f), + Path = new[] { new float2(0f, 1f), new float2(2f, 3f) }, + Cells = new List { new int2(1, 2) }, + }; + + var decoded = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(value, Options), Options); + + decoded.Position.Should().Be(value.Position); + decoded.Velocity.Should().Be(value.Velocity); + decoded.Path.Should().Equal(value.Path); + decoded.Cells.Should().Equal(value.Cells); + MessagePackSerializer.Serialize(new Carrier(), Options).Should().NotBeEmpty(); + + var absent = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(new Carrier(), Options), Options); + absent.Velocity.Should().BeNull(); + } + + private static void RoundTrip(T value) + { + var bytes = MessagePackSerializer.Serialize(value, Options); + MessagePackSerializer.Deserialize(bytes, Options).Should().Be(value); + } + } +} From 14621b3893c6a5fd9c0e3903eeea474ec25e091f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:15:29 +0200 Subject: [PATCH 02/64] Reference CultMath from Caching.MessagePack and scope the span helpers The first commit's resolver needs the CultMath project reference and scoped span parameters to compile; the golden test file also had two literal slips. Co-Authored-By: Claude Sonnet 5 --- .../CultMathResolver.cs | 20 +++++++++---------- .../GameCult.Caching.MessagePack.csproj | 1 + .../CultMathSerializationTests.cs | 7 ++++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultMathResolver.cs b/src/GameCult.Caching.MessagePack/CultMathResolver.cs index 71bdec2b..c854e12d 100644 --- a/src/GameCult.Caching.MessagePack/CultMathResolver.cs +++ b/src/GameCult.Caching.MessagePack/CultMathResolver.cs @@ -58,61 +58,61 @@ public sealed class CultMathResolver : IFormatterResolver new(typeof(T), new ArrayShape(write, read)); // Each reader takes the first dst.Length elements and skips the rest. - private static void Singles(ref MessagePackReader r, Span dst) + private static void Singles(ref MessagePackReader r, scoped Span dst) { for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) if (i < dst.Length) dst[i] = r.ReadSingle(); else r.Skip(); } - private static void Doubles(ref MessagePackReader r, Span dst) + private static void Doubles(ref MessagePackReader r, scoped Span dst) { for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) if (i < dst.Length) dst[i] = r.ReadDouble(); else r.Skip(); } - private static void Ints(ref MessagePackReader r, Span dst) + private static void Ints(ref MessagePackReader r, scoped Span dst) { for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) if (i < dst.Length) dst[i] = r.ReadInt32(); else r.Skip(); } - private static void Bools(ref MessagePackReader r, Span dst) + private static void Bools(ref MessagePackReader r, scoped Span dst) { for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) if (i < dst.Length) dst[i] = r.ReadBoolean(); else r.Skip(); } - private static void Bytes(ref MessagePackReader r, Span dst) + private static void Bytes(ref MessagePackReader r, scoped Span dst) { for (int i = 0, n = r.ReadArrayHeader(); i < n; i++) if (i < dst.Length) dst[i] = r.ReadByte(); else r.Skip(); } - private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + private static void Put(ref MessagePackWriter w, scoped ReadOnlySpan v) { w.WriteArrayHeader(v.Length); foreach (var e in v) w.Write(e); } - private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + private static void Put(ref MessagePackWriter w, scoped ReadOnlySpan v) { w.WriteArrayHeader(v.Length); foreach (var e in v) w.Write(e); } - private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + private static void Put(ref MessagePackWriter w, scoped ReadOnlySpan v) { w.WriteArrayHeader(v.Length); foreach (var e in v) w.Write(e); } - private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + private static void Put(ref MessagePackWriter w, scoped ReadOnlySpan v) { w.WriteArrayHeader(v.Length); foreach (var e in v) w.Write(e); } - private static void Put(ref MessagePackWriter w, ReadOnlySpan v) + private static void Put(ref MessagePackWriter w, scoped ReadOnlySpan v) { w.WriteArrayHeader(v.Length); foreach (var e in v) w.Write(e); diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index c2bd1c1b..a37232a9 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -6,6 +6,7 @@ + diff --git a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs index c6b6d161..76f95d34 100644 --- a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs +++ b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs @@ -28,9 +28,10 @@ namespace GameCult.Caching.Tests // Bytes the pre-collapse CultVec2/CultVec3/CultRect encodings and Aetheria's Float2/Int2/Bool2/Float3/Float4 // formatters produce: an array of components in declaration order, float32 as 0xCA. - [TestCase("92CA3F800000CA40000000")] - public void Float2_MatchesTheLegacyVectorBytes(string hex) + [Test] + public void Float2_MatchesTheLegacyVectorBytes() { + const string hex = "92CA3F800000CA40000000"; Hex(new float2(1f, 2f)).Should().Be(hex); Decode(hex).Should().Be(new float2(1f, 2f)); } @@ -102,7 +103,7 @@ namespace GameCult.Caching.Tests [Test] public void Matrices_AreArraysOfRows() { - Hex(new float2x2(1f, 2f, 3f, 4f)).Should().Be("9292CA3F800000CA400000009 2CA40400000CA40800000".Replace(" ", "")); + Hex(new float2x2(1f, 2f, 3f, 4f)).Should().Be("9292CA3F800000CA4000000092CA40400000CA40800000"); } [Test] From 20ac3b0b6ca95b4dcab951bd4ecb102e94039e18 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:18:09 +0200 Subject: [PATCH 03/64] Collapse CultVec2, CultVec3 and CultRect into CultMath float2, float3 and rect GameCult.Geometry no longer carries vector or rectangle values. CultCircle and CultSphere hold float2/float3, the isosurface and surface-nets extractors take float3 origin, and the per-call conversion to float3 is gone. Their MessagePack bytes are unchanged (pinned against the pre-collapse encodings), now produced by CultMathResolver through the default CultCache options. This is a breaking change to the GameCult.Geometry public API. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching.MessagePack/README.md | 1 + .../CultGeometryIsoSurface.cs | 5 +- .../CultGeometryPrimitives.cs | 229 ++---------------- .../CultGeometrySurfaceNets.cs | 7 +- src/GameCult.Geometry/README.md | 11 +- .../GeometryDocumentTests.cs | 63 +++-- .../IsoSurfaceTests.cs | 5 +- .../SurfaceNetsTests.cs | 68 +++--- 8 files changed, 93 insertions(+), 296 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index c69ab887..277d0cb2 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -9,6 +9,7 @@ format for the attribute-first `GameCult.Caching` stack. - `CultDocumentMessagePackSerialization` - `CultDocumentResolver` - `CultRecordRefFormatter` +- `CultMathResolver`: the canonical component-array encoding of every public CultMath value type, part of the default resolver chain Cult document payloads serialize through `MessagePackSerializer` with the options of the document's assembly (`CultDocumentMessagePackSerialization.OptionsFor`), diff --git a/src/GameCult.Geometry/CultGeometryIsoSurface.cs b/src/GameCult.Geometry/CultGeometryIsoSurface.cs index 583fdd87..b0967ece 100644 --- a/src/GameCult.Geometry/CultGeometryIsoSurface.cs +++ b/src/GameCult.Geometry/CultGeometryIsoSurface.cs @@ -39,7 +39,7 @@ namespace GameCult.Geometry public static CultGeometryTriangleMesh Extract( float[,,] samples, float isoValue = 0f, - CultVec3 origin = default, + float3 origin = default, float cellSize = 1f) { if (samples == null) throw new ArgumentNullException(nameof(samples)); @@ -56,7 +56,6 @@ namespace GameCult.Geometry var positions = new List(); var normals = new List(); var indices = new List(); - var originValue = new float3(origin.X, origin.Y, origin.Z); var cornerPositions = new float3[8]; var cornerValues = new float[8]; var tetraPositions = new float3[4]; @@ -66,7 +65,7 @@ namespace GameCult.Geometry for (var y = 0; y < samples.GetLength(1) - 1; y++) for (var z = 0; z < samples.GetLength(2) - 1; z++) { - var cubeOrigin = originValue + new float3(x, y, z) * cellSize; + var cubeOrigin = origin + new float3(x, y, z) * cellSize; for (var corner = 0; corner < CubeCorners.Length; corner++) { var offset = CubeCorners[corner]; diff --git a/src/GameCult.Geometry/CultGeometryPrimitives.cs b/src/GameCult.Geometry/CultGeometryPrimitives.cs index 99587b8e..dafcca68 100644 --- a/src/GameCult.Geometry/CultGeometryPrimitives.cs +++ b/src/GameCult.Geometry/CultGeometryPrimitives.cs @@ -1,197 +1,10 @@ using System; using System.Text.Json.Serialization; +using CultMath; using MessagePack; namespace GameCult.Geometry { - /// - /// Deterministic two-dimensional float vector for cross-runtime geometry and query contracts. - /// - [MessagePackObject] - public readonly struct CultVec2 : IEquatable - { - /// Creates a two-dimensional vector. - public CultVec2(float x, float y) - { - X = x; - Y = y; - } - - /// Gets the x component. - [Key(0)] - public float X { get; } - - /// Gets the y component. - [Key(1)] - public float Y { get; } - - /// Gets the zero vector. - [IgnoreMember] - [JsonIgnore] - public static CultVec2 Zero => new(0f, 0f); - - /// Gets the squared vector length. - [IgnoreMember] - [JsonIgnore] - public float LengthSquared => (X * X) + (Y * Y); - - /// Returns whether this vector is equal to another vector. - public bool Equals(CultVec2 other) => X.Equals(other.X) && Y.Equals(other.Y); - - /// - public override bool Equals(object? obj) => obj is CultVec2 other && Equals(other); - - /// - public override int GetHashCode() => HashCode.Combine(X, Y); - - /// - public override string ToString() => $"({X}, {Y})"; - - public static CultVec2 operator +(CultVec2 left, CultVec2 right) => new(left.X + right.X, left.Y + right.Y); - public static CultVec2 operator -(CultVec2 left, CultVec2 right) => new(left.X - right.X, left.Y - right.Y); - public static CultVec2 operator *(CultVec2 value, float scalar) => new(value.X * scalar, value.Y * scalar); - public static CultVec2 operator /(CultVec2 value, float scalar) => new(value.X / scalar, value.Y / scalar); - public static bool operator ==(CultVec2 left, CultVec2 right) => left.Equals(right); - public static bool operator !=(CultVec2 left, CultVec2 right) => !left.Equals(right); - } - - /// - /// Deterministic three-dimensional float vector for cross-runtime geometry and query contracts. - /// - [MessagePackObject] - public readonly struct CultVec3 : IEquatable - { - /// Creates a three-dimensional vector. - public CultVec3(float x, float y, float z) - { - X = x; - Y = y; - Z = z; - } - - /// Gets the x component. - [Key(0)] - public float X { get; } - - /// Gets the y component. - [Key(1)] - public float Y { get; } - - /// Gets the z component. - [Key(2)] - public float Z { get; } - - /// Gets the zero vector. - [IgnoreMember] - [JsonIgnore] - public static CultVec3 Zero => new(0f, 0f, 0f); - - /// Gets the XY plane projection. - [IgnoreMember] - [JsonIgnore] - public CultVec2 Xy => new(X, Y); - - /// Gets the XZ plane projection for Unity compatibility adapters. - [IgnoreMember] - [JsonIgnore] - public CultVec2 Xz => new(X, Z); - - /// Gets the squared vector length. - [IgnoreMember] - [JsonIgnore] - public float LengthSquared => (X * X) + (Y * Y) + (Z * Z); - - /// Returns whether this vector is equal to another vector. - public bool Equals(CultVec3 other) => X.Equals(other.X) && Y.Equals(other.Y) && Z.Equals(other.Z); - - /// - public override bool Equals(object? obj) => obj is CultVec3 other && Equals(other); - - /// - public override int GetHashCode() => HashCode.Combine(X, Y, Z); - - /// - public override string ToString() => $"({X}, {Y}, {Z})"; - - public static CultVec3 operator +(CultVec3 left, CultVec3 right) => new(left.X + right.X, left.Y + right.Y, left.Z + right.Z); - public static CultVec3 operator -(CultVec3 left, CultVec3 right) => new(left.X - right.X, left.Y - right.Y, left.Z - right.Z); - public static CultVec3 operator *(CultVec3 value, float scalar) => new(value.X * scalar, value.Y * scalar, value.Z * scalar); - public static CultVec3 operator /(CultVec3 value, float scalar) => new(value.X / scalar, value.Y / scalar, value.Z / scalar); - public static bool operator ==(CultVec3 left, CultVec3 right) => left.Equals(right); - public static bool operator !=(CultVec3 left, CultVec3 right) => !left.Equals(right); - } - - /// - /// Axis-aligned two-dimensional rectangle represented by canonical min and max corners. - /// - [MessagePackObject] - public readonly struct CultRect : IEquatable - { - /// Creates a rectangle and canonicalizes min/max corners. - public CultRect(CultVec2 min, CultVec2 max) - { - Min = new CultVec2(Math.Min(min.X, max.X), Math.Min(min.Y, max.Y)); - Max = new CultVec2(Math.Max(min.X, max.X), Math.Max(min.Y, max.Y)); - } - - /// Creates a rectangle and canonicalizes min/max corners. - public CultRect(float minX, float minY, float maxX, float maxY) - : this(new CultVec2(minX, minY), new CultVec2(maxX, maxY)) - { - } - - /// Gets the canonical minimum corner. - [Key(0)] - public CultVec2 Min { get; } - - /// Gets the canonical maximum corner. - [Key(1)] - public CultVec2 Max { get; } - - /// Gets the rectangle size. - [IgnoreMember] - [JsonIgnore] - public CultVec2 Size => Max - Min; - - /// Gets the rectangle center. - [IgnoreMember] - [JsonIgnore] - public CultVec2 Center => (Min + Max) * 0.5f; - - /// Returns whether the point lies inside or on the rectangle edge. - public bool Contains(CultVec2 point) - { - return point.X >= Min.X && - point.X <= Max.X && - point.Y >= Min.Y && - point.Y <= Max.Y; - } - - /// Returns whether this rectangle intersects another rectangle. - public bool Intersects(CultRect other) - { - return Min.X <= other.Max.X && - Max.X >= other.Min.X && - Min.Y <= other.Max.Y && - Max.Y >= other.Min.Y; - } - - /// Returns whether this rectangle is equal to another rectangle. - public bool Equals(CultRect other) => Min == other.Min && Max == other.Max; - - /// - public override bool Equals(object? obj) => obj is CultRect other && Equals(other); - - /// - public override int GetHashCode() => HashCode.Combine(Min, Max); - - /// - public override string ToString() => $"[{Min}..{Max}]"; - - public static bool operator ==(CultRect left, CultRect right) => left.Equals(right); - public static bool operator !=(CultRect left, CultRect right) => !left.Equals(right); - } - /// /// Two-dimensional circle for spatial queries. /// @@ -199,7 +12,7 @@ namespace GameCult.Geometry public readonly struct CultCircle : IEquatable { /// Creates a circle. - public CultCircle(CultVec2 center, float radius) + public CultCircle(float2 center, float radius) { if (radius < 0f) { @@ -212,7 +25,7 @@ namespace GameCult.Geometry /// Gets the circle center. [Key(0)] - public CultVec2 Center { get; } + public float2 Center { get; } /// Gets the circle radius. [Key(1)] @@ -221,29 +34,29 @@ namespace GameCult.Geometry /// Gets the circle bounds. [IgnoreMember] [JsonIgnore] - public CultRect Bounds => new( - Center.X - Radius, - Center.Y - Radius, - Center.X + Radius, - Center.Y + Radius); + public rect Bounds => new( + Center.x - Radius, + Center.y - Radius, + Center.x + Radius, + Center.y + Radius); /// Returns whether the point lies inside or on the circle edge. - public bool Contains(CultVec2 point) + public bool Contains(float2 point) { var delta = point - Center; - return delta.LengthSquared <= Radius * Radius; + return math.lengthsq(delta) <= Radius * Radius; } /// Returns whether this circle intersects a rectangle. - public bool Intersects(CultRect rect) + public bool Intersects(rect rect) { - var clampedX = Math.Max(rect.Min.X, Math.Min(Center.X, rect.Max.X)); - var clampedY = Math.Max(rect.Min.Y, Math.Min(Center.Y, rect.Max.Y)); - return Contains(new CultVec2(clampedX, clampedY)); + var clampedX = Math.Max(rect.min.x, Math.Min(Center.x, rect.max.x)); + var clampedY = Math.Max(rect.min.y, Math.Min(Center.y, rect.max.y)); + return Contains(new float2(clampedX, clampedY)); } /// Returns whether this circle is equal to another circle. - public bool Equals(CultCircle other) => Center == other.Center && Radius.Equals(other.Radius); + public bool Equals(CultCircle other) => Center.Equals(other.Center) && Radius.Equals(other.Radius); /// public override bool Equals(object? obj) => obj is CultCircle other && Equals(other); @@ -262,7 +75,7 @@ namespace GameCult.Geometry public readonly struct CultSphere : IEquatable { /// Creates a sphere. - public CultSphere(CultVec3 center, float radius) + public CultSphere(float3 center, float radius) { if (radius < 0f) { @@ -275,7 +88,7 @@ namespace GameCult.Geometry /// Gets the sphere center. [Key(0)] - public CultVec3 Center { get; } + public float3 Center { get; } /// Gets the sphere radius. [Key(1)] @@ -284,17 +97,17 @@ namespace GameCult.Geometry /// Gets the XY circle projection. [IgnoreMember] [JsonIgnore] - public CultCircle XyCircle => new(Center.Xy, Radius); + public CultCircle XyCircle => new(Center.xy, Radius); /// Returns whether the point lies inside or on the sphere surface. - public bool Contains(CultVec3 point) + public bool Contains(float3 point) { var delta = point - Center; - return delta.LengthSquared <= Radius * Radius; + return math.lengthsq(delta) <= Radius * Radius; } /// Returns whether this sphere is equal to another sphere. - public bool Equals(CultSphere other) => Center == other.Center && Radius.Equals(other.Radius); + public bool Equals(CultSphere other) => Center.Equals(other.Center) && Radius.Equals(other.Radius); /// public override bool Equals(object? obj) => obj is CultSphere other && Equals(other); diff --git a/src/GameCult.Geometry/CultGeometrySurfaceNets.cs b/src/GameCult.Geometry/CultGeometrySurfaceNets.cs index 45c3f1ec..bd78e053 100644 --- a/src/GameCult.Geometry/CultGeometrySurfaceNets.cs +++ b/src/GameCult.Geometry/CultGeometrySurfaceNets.cs @@ -44,7 +44,7 @@ namespace GameCult.Geometry public static CultGeometryQuadMesh Extract( float[,,] samples, float isoValue = 0f, - CultVec3 origin = default, + float3 origin = default, float cellSize = 1f) { if (samples == null) throw new ArgumentNullException(nameof(samples)); @@ -66,7 +66,7 @@ namespace GameCult.Geometry throw new ArgumentException("The isovalue must be finite.", nameof(isoValue)); } - if (!float.IsFinite(origin.X) || !float.IsFinite(origin.Y) || !float.IsFinite(origin.Z)) + if (!float.IsFinite(origin.x) || !float.IsFinite(origin.y) || !float.IsFinite(origin.z)) { throw new ArgumentException("The origin must be finite.", nameof(origin)); } @@ -80,7 +80,6 @@ namespace GameCult.Geometry var sizeY = samples.GetLength(1); var sizeZ = samples.GetLength(2); var cellCounts = new[] { sizeX - 1, sizeY - 1, sizeZ - 1 }; - var originValue = new float3(origin.X, origin.Y, origin.Z); var vertexIndex = new Dictionary<(int, int, int), uint>(); var positions = new List(); @@ -90,7 +89,7 @@ namespace GameCult.Geometry for (var cz = 0; cz < cellCounts[2]; cz++) { var cellCoord = new[] { cx, cy, cz }; - var vertex = CellVertex(samples, cellCoord, isoValue, originValue, cellSize); + var vertex = CellVertex(samples, cellCoord, isoValue, origin, cellSize); if (vertex == null) continue; vertexIndex[(cx, cy, cz)] = (uint)(positions.Count / 3); diff --git a/src/GameCult.Geometry/README.md b/src/GameCult.Geometry/README.md index 2d12dc25..9929e387 100644 --- a/src/GameCult.Geometry/README.md +++ b/src/GameCult.Geometry/README.md @@ -31,12 +31,9 @@ The package is deliberately substrate-shaped: ## Value Primitives The small value layer is for cross-runtime query contracts, physics probes, -native slices, and UI surfaces: +native slices, and UI surfaces. Vectors and rectangles are CultMath's `float2`, +`float3` and `rect`, and `GameCult.Caching.MessagePack` serializes them. -- `CultVec2` and `CultVec3`: whole-vector values for positions, velocities, - accelerations, and query inputs. SoA layouts may store them efficiently, but - the semantic API does not force callers to split x/y/z into unrelated fields. -- `CultRect`: canonical XY viewport/query rectangle stored as min/max corners. - `CultCircle`: 2D influence/query brush with rect intersection helpers. - `CultSphere`: 3D query primitive with whole-vector center and XY projection. @@ -85,8 +82,8 @@ cache miss; that is split-brain geometry. ## Documents -- `CultVec2`, `CultVec3`, `CultRect`, `CultCircle`, `CultSphere`: shared value - primitives for query and view contracts. +- `CultCircle`, `CultSphere`: shared value primitives for query and view + contracts. - `CultGeometryDomainDocument`: one hierarchical domain tree, suitable for Ragnarok/Fensalir-style feature DSL output. - `CultGeometryBuildRequest`: one LOD/frustum/budget request for workers. diff --git a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs index 302f53f3..daae992a 100644 --- a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs +++ b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs @@ -1,4 +1,5 @@ using FluentAssertions; +using CultMath; using GameCult.Caching; using GameCult.Caching.MessagePack; using GameCult.Mesh; @@ -19,54 +20,50 @@ namespace GameCult.Geometry.Tests [Test] public void GeometryPrimitives_CanonicalizeAndIntersect() { - var viewport = new CultRect(8f, 4f, -8f, -4f); + var viewport = new rect(8f, 4f, -8f, -4f); - viewport.Min.Should().Be(new CultVec2(-8f, -4f)); - viewport.Max.Should().Be(new CultVec2(8f, 4f)); - viewport.Center.Should().Be(CultVec2.Zero); - viewport.Contains(new CultVec2(2f, 3f)).Should().BeTrue(); - viewport.Intersects(new CultRect(7f, 3f, 10f, 5f)).Should().BeTrue(); - viewport.Intersects(new CultRect(9f, 5f, 10f, 6f)).Should().BeFalse(); + viewport.min.Should().Be(new float2(-8f, -4f)); + viewport.max.Should().Be(new float2(8f, 4f)); + viewport.center.Should().Be(float2.zero); + viewport.Contains(new float2(2f, 3f)).Should().BeTrue(); + viewport.Intersects(new rect(7f, 3f, 10f, 5f)).Should().BeTrue(); + viewport.Intersects(new rect(9f, 5f, 10f, 6f)).Should().BeFalse(); - var gravityBrush = new CultCircle(new CultVec2(10f, 0f), 3f); + var gravityBrush = new CultCircle(new float2(10f, 0f), 3f); gravityBrush.Intersects(viewport).Should().BeTrue(); - gravityBrush.Contains(new CultVec2(12f, 0f)).Should().BeTrue(); - gravityBrush.Contains(new CultVec2(14f, 0f)).Should().BeFalse(); + gravityBrush.Contains(new float2(12f, 0f)).Should().BeTrue(); + gravityBrush.Contains(new float2(14f, 0f)).Should().BeFalse(); } [Test] public void GeometryPrimitives_PreserveWholeVectorsForSoaAndPhysicsQueries() { - var sphere = new CultSphere(new CultVec3(1f, 2f, 3f), 5f); + var sphere = new CultSphere(new float3(1f, 2f, 3f), 5f); - sphere.Contains(new CultVec3(1f, 6f, 3f)).Should().BeTrue(); - sphere.Contains(new CultVec3(1f, 8f, 3f)).Should().BeFalse(); - sphere.Center.Xy.Should().Be(new CultVec2(1f, 2f)); - sphere.Center.Xz.Should().Be(new CultVec2(1f, 3f)); - sphere.XyCircle.Bounds.Should().Be(new CultRect(-4f, -3f, 6f, 7f)); + sphere.Contains(new float3(1f, 6f, 3f)).Should().BeTrue(); + sphere.Contains(new float3(1f, 8f, 3f)).Should().BeFalse(); + sphere.Center.xy.Should().Be(new float2(1f, 2f)); + sphere.Center.xz.Should().Be(new float2(1f, 3f)); + sphere.XyCircle.Bounds.Should().Be(new rect(-4f, -3f, 6f, 7f)); } + // The bytes CultCircle and CultSphere produced when they held CultVec2/CultVec3 (probed at 65311c9). [Test] - public void GeometryPrimitives_RoundTripThroughMessagePack() + public void GeometryPrimitives_KeepTheirPreCollapseBytes() { - var rect = new CultRect(new CultVec2(5f, -2f), new CultVec2(-1f, 7f)); - var payload = MessagePackSerializer.Serialize(rect); - var decoded = MessagePackSerializer.Deserialize(payload); + var options = CultDocumentMessagePackSerialization.Options; + var circle = new CultCircle(new float2(1f, 2f), 3f); + var sphere = new CultSphere(new float3(1f, 2f, 3f), 5f); - decoded.Should().Be(rect); - decoded.Min.Should().Be(new CultVec2(-1f, -2f)); - decoded.Max.Should().Be(new CultVec2(5f, 7f)); - } + Convert.ToHexString(MessagePackSerializer.Serialize(circle, options)) + .Should().Be("9292CA3F800000CA40000000CA40400000"); + Convert.ToHexString(MessagePackSerializer.Serialize(sphere, options)) + .Should().Be("9293CA3F800000CA40000000CA40400000CA40A00000"); - [Test] - public void GeometryPrimitives_JsonOmitsComputedConvenienceProperties() - { - var json = JsonSerializer.Serialize(new CultVec3(1f, 2f, 3f)); - - json.Should().Be("""{"X":1,"Y":2,"Z":3}"""); - json.Should().NotContain("LengthSquared"); - json.Should().NotContain("Xy"); - json.Should().NotContain("Xz"); + MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(circle, options), options) + .Should().Be(circle); + MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(sphere, options), options) + .Should().Be(sphere); } [Test] diff --git a/tests/GameCult.Geometry.Tests/IsoSurfaceTests.cs b/tests/GameCult.Geometry.Tests/IsoSurfaceTests.cs index 63b794b3..52f7219e 100644 --- a/tests/GameCult.Geometry.Tests/IsoSurfaceTests.cs +++ b/tests/GameCult.Geometry.Tests/IsoSurfaceTests.cs @@ -1,4 +1,5 @@ using System; +using CultMath; using FluentAssertions; using NUnit.Framework; @@ -77,8 +78,8 @@ namespace GameCult.Geometry.Tests samples[0, 1, 1] = 1f; samples[1, 1, 1] = 1f; - var first = CultGeometryIsoSurface.Extract(samples, origin: new CultVec3(10f, 20f, 30f), cellSize: 2f); - var second = CultGeometryIsoSurface.Extract(samples, origin: new CultVec3(10f, 20f, 30f), cellSize: 2f); + var first = CultGeometryIsoSurface.Extract(samples, origin: new float3(10f, 20f, 30f), cellSize: 2f); + var second = CultGeometryIsoSurface.Extract(samples, origin: new float3(10f, 20f, 30f), cellSize: 2f); second.Positions.Should().Equal(first.Positions); second.Normals.Should().Equal(first.Normals); diff --git a/tests/GameCult.Geometry.Tests/SurfaceNetsTests.cs b/tests/GameCult.Geometry.Tests/SurfaceNetsTests.cs index 3e3b3892..82c8fd43 100644 --- a/tests/GameCult.Geometry.Tests/SurfaceNetsTests.cs +++ b/tests/GameCult.Geometry.Tests/SurfaceNetsTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using CultMath; using FluentAssertions; using NUnit.Framework; @@ -73,28 +74,28 @@ namespace GameCult.Geometry.Tests for (var quad = 0; quad < mesh.QuadCount; quad++) { var edge = mesh.QuadEdges[quad]; - var insideCoord = new CultVec3(1f, 1f, 1f); - var edgeStart = new CultVec3(edge.X, edge.Y, edge.Z); + var insideCoord = new float3(1f, 1f, 1f); + var edgeStart = new float3(edge.X, edge.Y, edge.Z); var edgeEnd = OtherEndpoint(edge); // The single inside sample is (1,1,1); the edge's other endpoint is outside. var outsideCoord = edgeStart.Equals(insideCoord) ? edgeEnd : edgeStart; - var outward = Subtract(outsideCoord, insideCoord); + var outward = outsideCoord - insideCoord; var baseIndex = quad * 4; var v0 = Position(mesh.Positions, mesh.Quads[baseIndex]); var v1 = Position(mesh.Positions, mesh.Quads[baseIndex + 1]); var v2 = Position(mesh.Positions, mesh.Quads[baseIndex + 2]); var v3 = Position(mesh.Positions, mesh.Quads[baseIndex + 3]); - var cross = Cross(Subtract(v2, v0), Subtract(v3, v1)); + var cross = math.cross(v2 - v0, v3 - v1); // Restricted to non-degenerate (non-zero-area) quads: the spec's geometric // winding rule is only meaningful there. Degenerate/collapsed quads are covered // separately by AssertEdgeDirectionsBalance, since a zero cross product carries // no orientation information to check here. - if (cross.X == 0f && cross.Y == 0f && cross.Z == 0f) continue; + if (cross.Equals(float3.zero)) continue; - Dot(cross, outward).Should().BePositive($"quad for edge {edge} should wind outward"); + math.dot(cross, outward).Should().BePositive($"quad for edge {edge} should wind outward"); } } @@ -209,7 +210,7 @@ namespace GameCult.Geometry.Tests samples[1, 1, 1] = -1f; - var mesh = CultGeometrySurfaceNets.Extract(samples, origin: new CultVec3(10f, 20f, 30f), cellSize: 2f); + var mesh = CultGeometrySurfaceNets.Extract(samples, origin: new float3(10f, 20f, 30f), cellSize: 2f); mesh.VertexCount.Should().Be(1); mesh.Positions[0].Should().BeApproximately(10f + (2.9f / 3f * 2f), 1e-5f); @@ -387,10 +388,10 @@ namespace GameCult.Geometry.Tests // documented c0..c3 order is not swapped. var expected = new[] { - new CultVec3(1.5f, 0.5f, 0.5f), // c0 = (1, 0, 0) - new CultVec3(1.5f, 1.5f, 0.5f), // c1 = (1, 1, 0) - new CultVec3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) - new CultVec3(1.5f, 0.5f, 1.5f), // c3 = (1, 0, 1) + new float3(1.5f, 0.5f, 0.5f), // c0 = (1, 0, 0) + new float3(1.5f, 1.5f, 0.5f), // c1 = (1, 1, 0) + new float3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) + new float3(1.5f, 0.5f, 1.5f), // c3 = (1, 0, 1) }; AssertQuadCorners(mesh, edge, expected); @@ -413,10 +414,10 @@ namespace GameCult.Geometry.Tests // v3 trade places relative to the raw c0..c3 listing. var expected = new[] { - new CultVec3(0.5f, 1.5f, 0.5f), // c0 = (0, 1, 0) - new CultVec3(0.5f, 1.5f, 1.5f), // c3 = (0, 1, 1), swapped into v1 - new CultVec3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) - new CultVec3(1.5f, 1.5f, 0.5f), // c1 = (1, 1, 0), swapped into v3 + new float3(0.5f, 1.5f, 0.5f), // c0 = (0, 1, 0) + new float3(0.5f, 1.5f, 1.5f), // c3 = (0, 1, 1), swapped into v1 + new float3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) + new float3(1.5f, 1.5f, 0.5f), // c1 = (1, 1, 0), swapped into v3 }; AssertQuadCorners(mesh, edge, expected); @@ -438,10 +439,10 @@ namespace GameCult.Geometry.Tests // mirrored, so the documented c0..c3 order is not swapped. var expected = new[] { - new CultVec3(0.5f, 0.5f, 1.5f), // c0 = (0, 0, 1) - new CultVec3(1.5f, 0.5f, 1.5f), // c1 = (1, 0, 1) - new CultVec3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) - new CultVec3(0.5f, 1.5f, 1.5f), // c3 = (0, 1, 1) + new float3(0.5f, 0.5f, 1.5f), // c0 = (0, 0, 1) + new float3(1.5f, 0.5f, 1.5f), // c1 = (1, 0, 1) + new float3(1.5f, 1.5f, 1.5f), // c2 = (1, 1, 1) + new float3(0.5f, 1.5f, 1.5f), // c3 = (0, 1, 1) }; AssertQuadCorners(mesh, edge, expected); @@ -476,7 +477,7 @@ namespace GameCult.Geometry.Tests // balance regardless. var samples = Sphere(64); - var mesh = CultGeometrySurfaceNets.Extract(samples, origin: new CultVec3(1e4f, 1e4f, 1e4f), cellSize: 0.001f); + var mesh = CultGeometrySurfaceNets.Extract(samples, origin: new float3(1e4f, 1e4f, 1e4f), cellSize: 0.001f); mesh.QuadCount.Should().BeGreaterThan(0); AssertEdgeDirectionsBalance(mesh); @@ -651,7 +652,7 @@ namespace GameCult.Geometry.Tests { var samples = new float[2, 2, 2]; - Action act = () => CultGeometrySurfaceNets.Extract(samples, origin: new CultVec3(x, y, z)); + Action act = () => CultGeometrySurfaceNets.Extract(samples, origin: new float3(x, y, z)); act.Should().Throw(); } @@ -725,28 +726,17 @@ namespace GameCult.Geometry.Tests return usage; } - private static CultVec3 OtherEndpoint(CultGeometryGridEdge edge) => edge.Axis switch + private static float3 OtherEndpoint(CultGeometryGridEdge edge) => edge.Axis switch { - 0 => new CultVec3(edge.X + 1, edge.Y, edge.Z), - 1 => new CultVec3(edge.X, edge.Y + 1, edge.Z), - _ => new CultVec3(edge.X, edge.Y, edge.Z + 1), + 0 => new float3(edge.X + 1, edge.Y, edge.Z), + 1 => new float3(edge.X, edge.Y + 1, edge.Z), + _ => new float3(edge.X, edge.Y, edge.Z + 1), }; - private static CultVec3 Subtract(CultVec3 left, CultVec3 right) => - new(left.X - right.X, left.Y - right.Y, left.Z - right.Z); - - private static CultVec3 Cross(CultVec3 left, CultVec3 right) => new( - (left.Y * right.Z) - (left.Z * right.Y), - (left.Z * right.X) - (left.X * right.Z), - (left.X * right.Y) - (left.Y * right.X)); - - private static float Dot(CultVec3 left, CultVec3 right) => - (left.X * right.X) + (left.Y * right.Y) + (left.Z * right.Z); - - private static CultVec3 Position(float[] positions, uint index) + private static float3 Position(float[] positions, uint index) { var offset = (int)index * 3; - return new CultVec3(positions[offset], positions[offset + 1], positions[offset + 2]); + return new float3(positions[offset], positions[offset + 1], positions[offset + 2]); } // Hand-derived (double precision, independent of Interpolate) expected position for a @@ -760,7 +750,7 @@ namespace GameCult.Geometry.Tests return (float)(amount / 3d); } - private static void AssertQuadCorners(CultGeometryQuadMesh mesh, CultGeometryGridEdge edge, CultVec3[] expected) + private static void AssertQuadCorners(CultGeometryQuadMesh mesh, CultGeometryGridEdge edge, float3[] expected) { var quadIndex = Array.IndexOf(mesh.QuadEdges, edge); quadIndex.Should().BeGreaterThanOrEqualTo(0, $"a quad for edge {edge} should exist"); From 1e3a337b8152ad3523eb4b3943c792fcbd020f09 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:35:33 +0200 Subject: [PATCH 04/64] Record C2a Soul hold, the tripwire and duplicate-id rulings, and the C2b precondition Co-Authored-By: Claude Opus 5.5 --- docs/document-variants-cut.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index 680c8440..923c1735 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -314,6 +314,31 @@ not closeable, and a fix batch is in Hands: **Open for the operator:** plain records with duplicate unique-index values are silently overwritten today. That predates variants. Fixing it could refuse existing stores. +**C2a status, 2026-09-30.** Hands pushed `hands/variants-c2a` at `640ad9a`. +- Element ids are 12 hex characters. Writes mint them at random. Load mints them deterministically. + Derived ids use `[CultElementId(nameof(Member))]`. +- The CDN chunk ref's id is its offset. The witness types carry ids, and the ids cross C#, TS, Python and + Rust. + +Soul held it. Fixes are in Hands: +- ids are never minted inside variant overrides; +- interface unions are never minted; +- a null derived source leaves an empty id; +- load minting is culture-dependent; +- the minted-on-load list goes stale; +- a refused batch leaves minted ids behind; +- the id format is unchecked; +- late union-slot reuse passes registration. + +**C2a rulings (operator, 2026-09-30):** +- **Old readers are tripwired.** A store that holds element ids carries a marker that every pre-C2a reader + refuses under C0's unknown-header rule. An old C# reader can therefore never silently strip nested ids. + MessagePack's generated formatters skip unknown slots inside elements. +- **Duplicate element ids refuse at load,** naming the record, the list and the id. This matches the + duplicate-index ruling. +- **C2b precondition (Soul F7).** Whether a pre-id element keeps its load-time id depends on the write path. + C2b must refuse an override that targets a record whose ids exist only in memory, or mint on open. + **Operator rulings, 2026-09-30:** - **Duplicate unique-index values do not load** ("Duplicate indices should not load"). This covers plain records too, so R6-for-indexes becomes one general rule: From 2dc7c122f9018e4c2e06806c8afa79ee0e7f47d6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:36:18 +0200 Subject: [PATCH 05/64] Map CultCache publication: order as data via a journal under the cache gate Imagination rebuilt the writer-thread scheduler Soul pass 3 proposed: it deadlocks exactly as bcae483 did (deleted in 4562340). The map keeps per-writer delivery, validates getters before minting, isolates observers, and moves the shard log onto a gate-held journal. Q-P1..Q-P4 open. Co-Authored-By: Claude Opus 5.5 --- docs/cultcache-publication-cut.md | 638 ++++++++++++++++++++++++++++++ 1 file changed, 638 insertions(+) create mode 100644 docs/cultcache-publication-cut.md diff --git a/docs/cultcache-publication-cut.md b/docs/cultcache-publication-cut.md new file mode 100644 index 00000000..f13ba680 --- /dev/null +++ b/docs/cultcache-publication-cut.md @@ -0,0 +1,638 @@ +# CultCache publication: the cache owns "published once", and order stays data + +Status: cut map, Imagination pass 1, 2026-09-29. It is not committed yet. Self +commits it to `main` as `docs/cultcache-publication-cut.md`. The ends are the +operator ruling of 2026-09-30: every committed cache change reaches subscribers +exactly once. This document owns the means. + +- **Body.** CultLib `main` `65311c9ad770828adafe40c2c14b2d3da517628e`. Every + `file:line` is against it unless it names the branch. +- **Superseded attempt.** `origin/hands/cultnet-publish-commits`, tip + `28248eb`, base `f9f107a`. Soul pass 3 said do not merge it, and that stands. + - Main has not touched `CultCache.cs`, `CultNetDatabase.cs`, the composition + doc or the networking tests since `f9f107a`: `git diff --stat` is empty. + - `git merge-tree 65311c9 28248eb` is clean. +- **Scope.** The C# cache's publication and CultNetDatabase's use of it. There + is no wire-format change and no other runtime changes (§6). + +## The finding that shapes this map + +The direction Imagination was handed was: "ordered delivery belongs in the +cache's own publication, on the writer's thread, so a writer delivers its own +changes before returning". That design **has been built before, and it was +deleted**. + +- `bcae483` (2026-09-13 20:09) made the cache a delivery scheduler: tickets, a + delivery gate, turn waits and per-thread queues for reentrant writes. +- `4562340`, 34 minutes later, deleted it: "It deadlocked inside CultLib: a + load's delivery reaching a CultNetDatabaseSubscriptionServer observer that + needs _lifecycleGate, while a subscribe's demand handler holding that gate + writes the cache and waits for the delivery's turn." +- It replaced it with **order as data**: a per-cache `Sequence`, and no + delivery scheduler. +- The regression tests it left, `ObserverWritingTheSameCacheDoesNotWaitOnOtherDeliveries` + (`ConditionalCommitTests.cs:763`) and + `SubscriptionDemandHandlerWritingCacheDuringDeliveryDoesNotDeadlock` + (`NetworkingTests.cs:2244`), are live on main. + +This pass rebuilt that scheduler as a prototype. It is the per-cache turnstile +with same-thread trampolining at `79f4639`, and it reproduces the deletion: + +- Caching tests: 278 of 279 pass, and `ObserverWritingTheSameCacheDoesNotWaitOnOtherDeliveries` + fails. +- The networking suite hangs, and Yggdrasil's idle watchdog killed it at 900 s. + The blame-hang run that names the hung test is recorded below. + +The reason is structural, not an implementation slip. Once a writer waits for +another thread's delivery, any lock that an observer takes and a writer holds +becomes a deadlock: + +- total cross-thread order; +- a writer delivering before it returns; +- no writer waiting on another thread. + +Pick two. The branch's drainer chose order plus no-wait, and lost +before-return (F3 and F4). The turnstile chose order plus before-return, and +lost no-wait (the `bcae483` deadlock). + +**So this map keeps order as data (the `4562340` ruling) and moves the one +order-sensitive consumer, the CultNet mutation log, to the place order is +decided: under the cache gate.** + +- Every writer keeps delivering its own changes, on its own thread, before it + returns, exactly as main does. F3 is satisfied by not regressing it. +- The log becomes a **journal** written in the admitting hold, in Sequence + order. +- The reorder buffer and drainer die, and no scheduler replaces them. This + departs from Soul's stated direction on purpose, and **Q-P1** puts it to the + operator. + +## Probe evidence + +All of this ran on Yggdrasil via `ygg-verify.sh`, from scratch commits in a +throwaway worktree that no branch holds. + +- Probe file: `tests/GameCult.Caching.Tests/PublicationProbeTests.cs` in scratch commit `79f4639`. The worktree has been removed. The commit is unreferenced in the local object store, where gc can collect it; it also survives on the Yggdrasil mirror as `refs/verify/79f4639f8f5b95db7ce16f244a081c1ec101a468`. +- `9c5abf5` is the probes on main. `79f4639` is the same probes plus the + turnstile prototype in `CultCache.cs`. + +| # | Probe | main `65311c9` | Turnstile prototype `79f4639` | +|---|---|---|---| +| Q1 | A batch `{ok, boom}` where boom's `[CultName]` getter throws | `TargetInvocationException`. `_sequence` goes from 0 to 1, because ok was minted. **Both ok and boom are in memory**, and `GetByName("ok")` finds ok. **Nothing is published**, and the next write gets Sequence 2. F1 is worse than stated: memory is corrupt and unpublished. In a store-backed cache, `land` (`:2325`) already ran, so the store holds it too. | No stall, but memory is still corrupt. | +| Q2 | Two `Watch` observers, the first throws | With the default R3 handler, the second observer sees 2 of 2 and the writer sees nothing. **With a rethrowing handler, the second observer sees 0 of 2 and every writer gets the exception.** The comment at `CultCache.cs:2374`, "OnNext does not throw", is false under fail-fast. | With its own observer array and a per-observer catch, the second observer sees 2 of 2 and the writer gets the exception afterwards. | +| Q3 | Writer A's observer blocks on `a`, then B writes `b` | The recorder sees `b(2)` before `a(1)`. There is no cross-thread order, by design since `4562340`. | Sees `a` then `b`, because B waited for A. | +| Q4 | An observer on `x` writes `y` | `y(2)` before `x(1)`: nested, depth-first, and the nested write returns after `y` is published. | `x` then `y`, and the nested write returns before `y` is published. | +| Q5 | `OnUpdate` on Upsert and on Commit | 0 fires, so loads only. `Watch` sees 2. | Unchanged. | +| Q6 | 8 threads × 1000 writes, plus reentrant writes | 67 ms, 2645 of 8800 cross-thread inversions, and 0 writes returned undelivered. | 516 ms, 0 inversions, 0 undelivered. | +| Q7 | Soul's P2 shape: a handler waits 1 s for another thread's write | Observed in time. | Not observed in time. | +| Suites | Caching, Networking and Mesh | (green on main) | Caching fails `ObserverWritingTheSameCacheDoesNotWaitOnOtherDeliveries`, and Networking **hangs** until the 900 s idle kill. Rerun with `--blame-hang-timeout 90s`: 286 passed, then the test host hung and was dumped. The blame sequence file was not extracted, so **the hung test is not named**. It is consistent with, but not proven to be, `SubscriptionDemandHandlerWritingCacheDuringDeliveryDoesNotDeadlock`, whose own comment says "disposal would hang too". | + +## Authority map + +- **Owner.** `CultCache`. The hold that mints a Sequence range also: + 1. journals the range under the gate, in Sequence order (new); + 2. publishes the range to every observer, isolated, on the minting writer's + thread, after it releases the gate and before it returns (main's model, + now isolated). + + Nothing waits on another call's delivery. +- **Inputs.** + - The changes returned by `Apply` (`CultCache.cs:2707`), the single mutation + site. + - The pre-mint projection of names and indexes (Cut 1). + - The journal list, the observer array and `OnUpdate`. +- **Outputs.** + - Journal calls: one per admission, under the gate, in Sequence order. + - `Watch` / `WatchRecord` notifications that carry `Sequence`. + - `OnUpdate` for loads. + - Observer and journal exceptions, rethrown to the writer. +- **Derived state.** + - `Sequence` is order as data. + - CultNet's primary log sequence is minted in the journal, so log order is + cache order. + - A replica's log sequence is copied from the primary entry. + - The database change kind (`Predicted`, `Reconciled`, `Added`, `Updated`, + `Removed`) is decided in the journal. + - `CultNetDatabaseChange.Sequence` (new) is copied from the cache change. + - Wire projections drop stale by it (Cut 6). +- **Forbidden writers.** + - Nothing re-sequences or schedules cache changes. The branch's `_parked`, + `_ready`, `_nextSequence`, `_draining`, `PendingChange` and the drain body + of `OnCacheChange` die, and so do the turnstile, `bcae483` and the + prototype. + - No R3 `Subject` fans out cache changes (Q2). + - `Apply` and `Index` run no user code. + - Doors never read or write prediction state: the branch's `HasPrediction` + and `_predictionGate` die. + - The database's Watch observer never logs; only the journal logs. + - A journal never enters the cache. +- **Shared paths.** Every path goes `Held` → `Admit` → `Resolve` → **Project** + → `land` → `Apply` → **journal** → leave gate → **publish**: + - loads: `AddBackingStore` `:1909`, `Loaded` `:1940`, `PullAll*` `:1959` and + `:3052`; + - `Commit` / `TryCommit` (`Land`, `:2272`); + - `UpsertAsync`, `UpsertVariantAsync`, `FlattenAsync` and `Remove` + (`:2004-2056`, `Write` `:2252`); + - a variant dependent's re-resolution (`VariantPlan.Dependents`, in the same + `Apply`); + - direct calls on an attached store (`:2924`); + - `Dispose` (`:2142`); + - the database doors, replica apply and snapshot resync, which only write + the cache and hand context to the journal. + + The single exception is the replica's absent-key delete, which has no cache + change. It logs directly (§4). +- **Deletion line.** + - In the cache: `Subject _changes` (`:1809`), the `_changes.OnNext` + call and its false comment (`:2374-2375`), `_changes.Dispose()` + (`:2154`), and the accessor calls in `Index` (`:2798-2804`). + - In the database, on the salvaged branch: the reorder buffer and drainer, + `_predictionGate` and `HasPrediction`, the logging in `Deliver`, and the + `Subject _changes` fan-out. + +## The design, question by question + +### 1. Validation before minting: a refused change mints nothing + +- **Where.** Add **Project** to `Admit` (`:2313`), between `Resolve` (`:2324`) + and `land` (`:2325`). +- **What.** Over `plan.Admitted ∪ plan.Dependents`, it evaluates once per + record: `NameAccessor(document)` and every `IndexAccessors[alias](document)`. + It keeps `(Name, (Alias, Value)[])` per key on the plan. +- **A throwing getter refuses the whole admission.** It raises an + `InvalidOperationException` that names the key, the member and the unwrapped + message; today the caller sees a bare `TargetInvocationException`. The + refusal comes before `land`, so the admission writes no store, touches no + memory and mints nothing. +- **Readers take the projection, not the document.** + - `Index` (`:2793-2806`). + - `RefuseVariantIndexSharing` (`:2659-2660`). + - The variant-name rule in `ResolveOne` (`:2616`). Its ancestor read at + `:2621` reads held documents, pre-land, and may stay. + + `Resolve` currently runs the refusal rules before any projection exists. + Hands makes Project the first pass over the resolved plan and moves both + rules after it, so rules read the projection and only the projection reads + documents. +- **Apply becomes user-code-free.** It then does only dictionary and + `ConditionalWeakTable` work on precomputed data. +- **Loads refuse too.** A store whose record's getter throws fails to open, + loudly. That is the rule "hydration failure on open is loud" (composition doc + `:88-92`). Today it half-indexes silently. + +### 2. Observer isolation + +- **Replace the `Subject` with the cache's own observer array**, + copy-on-write under a small lock. + - `Watch` (`:1887`) becomes `Observable.Create(register)` followed + by the existing `Where` and `Select`. + - `Dispose` calls `OnCompleted` on each observer, which is what + `Subject.Dispose` did. +- **Publish loops change → observer, and wraps each `OnNext` in its own + try/catch.** + - R3's `Observer` already catches handler exceptions (`OnErrorResume`, then + the unhandled handler). So an exception reaches the cache only when fail-fast + was asked for: a rethrowing `ObservableSystem` handler, or a custom observer. + - Escaped exceptions join the `OnUpdate` failures. They are rethrown to the + writer after every observer has received every change of the hold. That is + the existing `OnUpdate` policy (composition doc `:77-82`). + - The observer stays subscribed (**Q-P2**). The commit stands either way. +- **Prototype.** At `79f4639` (Q2) this took about 15 lines. Hands keeps that + part and drops the turnstile around it. + +### 3. Delivery thread, order and reentrancy + +- **Unchanged from main, on purpose.** Each outermost hold publishes exactly its + own changes, on its thread, after it leaves the gate and before it returns + (composition doc `:59-65`, `Held` `:2339`). + - A write from inside a handler is its own outermost hold. It delivers + depth-first before returning, as Q4 shows on main. + - There is no cross-thread order. Every change carries `Sequence`, and + latest-state consumers drop stale changes by it (doc `:68-73`, and the Mesh + mirrors at `CultMesh.cs:1534-1540`). +- **Why this is not a regression of the ruling.** The ruling asks for + exactly-once, not for a total delivery order. + - Exactly-once in the cache comes from the single mint site plus isolation + plus Cut 1 (§1-2). + - Order is needed by exactly one consumer, the shard log. It gets that order + from the journal (§4), not from delivery. +- **The journal is the new mechanism.** + - Signature: `IDisposable CultCache.AddJournal(Action>>)`. + Its name and shape are **Q-P3**. + - It is called in `Admit` right after `Apply` (`:2328-2329`), under the gate, + once per admission, with that admission's changes in Sequence order. + Journals run in registration order. + - A journal that enters the cache, meaning any hold on this cache while a + journal runs, throws `InvalidOperationException`. It is guarded by a flag + checked in `Held`'s `Monitor.IsEntered` branch (`:2341`). This is the + store rule ("a store call must not wait for a write to its own cache"), + enforced. + - Journal exceptions follow the observer policy: the admission stands, it is + still published, and the exception is rethrown to the writer after + publication. + - Adding or removing a journal takes the gate, so registration is atomic with + respect to admissions. +- **Lock order.** Cache gate, then journal-internal leaf locks (the database's + `_logGate`). Observers still run outside the gate, so the `bcae483` cycle + (an observer needing `_lifecycleGate` while a gate holder writes the cache) + cannot form. No journal takes `_lifecycleGate`, and no writer waits on a + delivery. +- **Soul's P2 passes.** A handler waiting on another thread's write sees it, + because nothing orders deliveries (Q7 on main). + +### 4. What CultNetDatabase keeps + +The database is rebuilt on the journal. It has no queue. + +- **Journal (under the cache gate, per admitted change, in Sequence order):** + 1. `TakeDoor(change)`, with the shape rule salvaged from the branch + (`:1219-1234`). A put door matches a change whose new document is its + instance and whose previous document is not. A removal door matches a + change whose previous document is its instance and which has no new one. + - The consumption now happens inside the door's own hold, before any + observer runs. So an observer's later write to the door's key or + instance can never wear the context (the `f3343c8` defect), and + unconditional `Dispose`-removal stays correct. + 2. Predictions. A `Predicts` door adds the prediction. On an `Authoritative` + door (`ApplyPutAsync`, replica put), if a prediction exists for the key, + the kind is `Reconciled` and the prediction is removed. + - The gate serializes this, so `_predictionGate` and `HasPrediction` are + deleted (F5). + - Soul's P6 cannot happen: the prediction is recorded before + `PutPredictedAsync` returns. + 3. Primacy. + - On a shard primary, where the door is not `NoLog` (prediction, snapshot): + build the log entry, including the wire message from a `Put`/`Delete` + door or `ToLogEntryMessage`, and **only then** mint the per-shard + sequence and append it to memory and to the durable store. A + serialization failure then mints nothing, so the log can never hold a + gap. + - On a replica, record the primary's entry from the `Replica` door, + under its own sequence. + - For a local write on a replica, log nothing. + 4. Stash `(cache Sequence → CultNetDatabaseChange)` under `_logGate`, for the + publisher. +- **Watch observer (after the gate, on the writer's thread, before it + returns):** it removes the stash entry for `change.Sequence` and publishes it. + - A change with no stash was admitted before the database existed and is + ignored. + - Register the observer first and the journal second, in the constructor, so + a stash can never be orphaned. +- **Log-before-publish** is structural: the journal runs before the gate opens. +- **The replica's absent-key delete (F6)** logs directly under `_logGate`. It + is in order because every earlier entry of the same pull loop was journaled + inside its own hold, before that `UpsertAsync`/`Remove` returned. This holds + even when a pull runs inside a handler, since nested holds journal + synchronously. +- **The in-memory log is keyed and ordered by sequence.** It becomes a + `SortedList` per shard, to match the durable store + (`CultNetShardMutationLogStorage.cs:55-64, 75-89`: that store replaces by + sequence and orders by sequence). +- **Kept from the branch:** + - the door table keyed by instance identity, and the shape rule; + - "only a primary mints; replicas publish"; + - log-before-publish; + - the snapshot resync "logs nothing" rule. Its removal door (branch `:733`) + must set `Removal = true`, or the shape rule never matches it. +- **Database fan-out isolation.** The database's `Subject _changes` + (main `:487`) has the Q2 defect one layer up: the subscription server's + per-peer handlers sit on it. Replace it with the same isolated + observer-array pattern as §2 (about 20 lines). Its exceptions reach the + writer through the cache's own per-observer catch. +- **`CultNetDatabaseChange` gains `long Sequence`**, the cache change's + Sequence, as an optional constructor argument, exactly as + `CultCacheDocumentChange` did (composition doc `:57-58`). Without it, + in-process latest-state consumers cannot drop stale changes. +- **Gone:** F3 (no deferred delivery), F4 (no drainer), F7 (no queue to + strand) and F8 (no guard). + +### 5. Backpressure and memory bounds + +- **No buffer exists.** A writer delivers its own changes, and they are held + only in its `mine` list for the length of its own call. +- **The journal stash** holds at most the changes of holds between their + journal and their publication. That is one hold per writing thread, and each + entry is removed by the same thread that added it. +- **A slow observer slows only the writer whose change it is handling.** That + is main's behaviour. It does not slow other writers, and it does not grow + memory. This replaces the branch's unbounded drainer backlog (F4, Soul P5). +- **The journal runs under the gate, so a slow journal slows every writer.** + - The database journal does in-memory work plus one durable + `mutationLogStore.Append` per logged change. `Append` rewrites the shard + file (`CultNetShardMutationLogStorage.cs:83-89`). + - That is I/O under the gate, which the locking doctrine already allows for + stores ("A store doing I/O on behalf of its cache ... holds the gate", + composition doc `:98-101`). It is still a new cost, so Cut 4's + verification measures commit latency with a file log store. + - The whole-file rewrite is a pre-existing monolith. It is recorded as a + follow-up, not fixed here. + +### 6. Cross-runtime impact + +The Eyes survey was run at `65311c9`. + +- **`cultcache-rs`, `cultcache-ts` and `cultcache-py` have no publication + surface:** no watch, no listener and no Sequence. + - Rust: `lib.rs:1970-1975, 2213-2420`. + - TypeScript: `cult-cache.ts:164-181`, "Observer delivery is not scheduled + here". + - Python: `cache.py:30-45`, which refuses reentrant mutation. +- **`cultnet-rs`, `cultnet-ts` and `cultnet-py` have no CultNetDatabase + equivalent and no cache watcher.** The only server-side shard log is the + Python interop test peer (`interop_peer.py:565, 583`), which mints + `len(log)+1` under its own lock, inline with the write. That is already the + journal's shape. +- **Wire parity is unaffected.** + - Cache `Sequence` never goes on the wire (composition doc `:47-49`). + - `CultNetShardLogEntryMessage` does not change. + - `CultNetDatabaseChangeRawMessage` (`CultNetSchemaMessages.cs:1016`) carries + no sequence, and this map adds none: stale protection for wire + subscriptions is server-side (Cut 6). + - No vector in `tests/vectors/` encodes publication order. +- **No other runtime must change.** +- **Follow-up.** `cultmesh-py`'s `CultMeshDatabase` watch (`node.py:421-505, + 1059-1065`) has F2 (a throwing callback skips the rest and reaches the + writer), and it has no Sequence. When Python grows cache publication, it + takes this contract. + +## Cuts + +Every cut is on CultLib, branch `hands/cultcache-publication` from `main` +`65311c9`, in a worktree Self creates. Cuts 1-3 touch only the cache and land +first. Cuts 4-6 depend on Cut 3. Heavy verification runs on Yggdrasil. + +**The negative checks that stay green in every cut** are the `4562340` tests: + +- `ObserverWritingTheSameCacheDoesNotWaitOnOtherDeliveries`; +- `SubscriptionDemandHandlerWritingCacheDuringDeliveryDoesNotDeadlock`; +- `SequenceIncreasesInAdmissionOrder`; +- `WatchRecordIgnoresStaleSequence`. + +They are the proof that no scheduler has come back. + +### Cut 1. Project before mint (cache, behaviour) + +- **First.** Commit Q1 as a failing test named for the rule. +- **Deletes first.** + - Accessor calls in `Index` (`:2798-2804`). + - Direct accessor reads in `RefuseVariantIndexSharing` (`:2659-2660`) and + `ResolveOne` (`:2616`). +- **Adds.** + - `Project` in `Admit` (`:2324-2325`). + - A per-key projection on `VariantPlan` (`:2456`). + - The early returns in `Resolve` (`:2477-2478`, `:2530-2534`) still pass + through `Project`, because it is called from `Admit`. +- **Authority.** Project owns "this admission's name and index values", and + the rules and `Index` read them. `Apply` runs no user code. +- **Verification.** + - Tests: + - **A refused getter mints nothing and changes nothing.** `GetWithSequence` + is unchanged, both keys are absent, the name lookup misses, nothing is + published, and the next write is previous+1. + - **Store-backed batch:** the store file is byte-identical, and a reopen + holds neither key. + - **A load whose getter throws** refuses the open and leaves the file + byte-identical. + - **Once per landed record:** a counting getter runs exactly once per + landed record, including a variant dependent re-resolved by a base edit. + - **The refusal names** the key and the member. + - Soul P4. + - Negative grep: `rg -n "NameAccessor\?\.Invoke|IndexAccessors" src/GameCult.Caching/CultCache.cs` + has no hit in `Apply`, `Index` or `Unindex`. `:315` is store-side and + legitimate. + - Stryker.NET on the diff. +- **Ledger.** About −15 and +45. + +### Cut 2. Observer isolation (cache, behaviour) + +- **Deletes first.** + - `Subject _changes` (`:1809`). + - `_changes.OnNext` and the false comment (`:2374-2375`). + - `_changes.Dispose()` (`:2154`). +- **Adds.** + - The observer array and `Observable.Create` in `Watch` (`:1887-1897`). + - The per-observer catch in `Publish` (`:2369`), with escapes joining + `failures`. + - `OnCompleted` on `Dispose`. +- **Docs.** Composition doc `:77-82`: escaped observer exceptions follow the + `OnUpdate` policy. +- **Verification.** + - Tests: + - **Q2 with a rethrowing handler:** the second observer receives every + change. The writer gets the escaped exception after delivery, and the + thrower is still subscribed (per Q-P2). + - **Subscription order:** subscribing and unsubscribing during a delivery + leaves every other observer's stream complete. + - **Dispose** completes observers. + - Soul P3. + - The four `4562340` tests stay green. + - Stryker on the diff, especially removing the catch and breaking out of + the loop on the first failure. +- **Ledger.** About −5 and +30. + +### Cut 3. The journal (cache, behaviour) + +- **Adds.** + - `AddJournal` (Q-P3), with registration under the gate. + - The invocation in `Admit` after `Apply` (`:2328`). + - The no-reentry guard in `Held` (`:2341`). + - Exceptions routed like observer failures. +- **Docs.** A composition-doc paragraph in "Store composition" and in + "Locking": journals run under the gate, in Sequence order, and must not + enter the cache or wait for anything that needs it. +- **Verification.** + - Tests: + - **Order:** 8 threads × 1000 writes. The journal sees contiguous, + strictly increasing Sequences, and each call's changes arrive together. + - **Timing:** the journal runs before any observer of the same change, and + after `Apply` (`Get` inside the journal sees the new document). + - **Reentry:** a journal that writes the cache throws, and the admission + stands. + - **Refusal:** a refused admission journals nothing. + - **Dependents:** a variant dependent is journaled in the same call as its + base edit. + - **Failure:** a throwing journal still lets the change publish, and the + writer gets the exception. + - The `4562340` tests stay green. + - Stryker on the diff. +- **Ledger.** About +50. + +### Cut 4. Salvage the branch and rebuild CultNetDatabase on the journal (networking) + +- **4a (history).** `git merge origin/hands/cultnet-publish-commits` into the + cut branch. It is clean. The merge carries the tests; the queue dies in 4b. + Self may choose a squash-salvage instead, with the same content. +- **4b (subtraction, then behaviour).** + - **Deletes first**, at branch `28248eb` `CultNetDatabase.cs`: + - `_parked`, `_ready`, `_nextSequence` and `_draining` (`:502-505`); + - the handshake at `:530`; + - `OnCacheChange`'s queue body (`:1083-1135`); + - `PendingChange` (`:1265-1275`); + - `_predictionGate` and `HasPrediction` (`:506-507`, `:1209-1213`) and + their call sites (`:916`, `:1574`); + - the logging in `Deliver` (`:1168-1201`). + - **Moves.** Logging, predictions and door consumption move into the journal + (§4). `Deliver` becomes the publisher that reads the stash. + - **Changes.** + - Door flags: `Predicts` and `Authoritative`. + - The snapshot removal door sets `Removal`. + - The in-memory log becomes a `SortedList`. + - **Keeps.** Door identity and the shape rule, primacy, `_logGate` for + readers, and the absent-key delete's direct log. +- **Verification.** + - Every salvaged test passes: + - `CultNetPublishesCommitsTests` (13); + - `CultNetPublicationOrderTests`: S1, S1b, S2, S3, S4, S6, S7, + log-before-publish, and in-place re-upsert. + - Soul's P1-P7 from `SoulPub3Tests.cs` are committed, all passing. P2 passes + because nothing orders delivery. + - New tests: + - a reentrant `database.PutAsync` from inside a database handler is logged + with its context and published once; + - a reentrant `ApplyShardLogResponseAsync` from inside a handler keeps the + replica log in primary order; + - a primary serialization failure mints no log sequence. + - Commit latency with `CultNetFileShardMutationLogStore`, main against Cut 4. + It is reported, not gated. + - Negative grep: `rg -n "_parked|_ready|_draining|_nextSequence|_predictionGate|HasPrediction" src/GameCult.Networking` + returns nothing. + - The `4562340` tests stay green. + - Stryker on the diff. +- **Ledger.** Against the branch, about −120 and +60. + +### Cut 5. Database fan-out isolation and Sequence (networking, behaviour) + +- **Deletes first.** The database's `Subject _changes` (main `:487`). +- **Adds.** + - The isolated observer array. + - `CultNetDatabaseChange.Sequence` (main `:319-371`), set by the journal. +- **Verification.** + - With a rethrowing handler and one throwing database subscriber, the other + subscriber and a live subscription-server peer receive every change. + - The disposed-database test. + - `Sequence` equals the cache change's. +- **Ledger.** About −10 and +30. + +### Cut 6. Stale protection on wire projections (networking, behaviour) + +This is a separate cut that closes a pre-existing hole. Composition doc +`:50-53` says the CultNet streams "have no ordering guarantee and no stale +protection", and the branch's total order was reaching for this. + +- **The rule.** A projection sends a record only if the change's `Sequence` is + above the Sequence it last applied for that key. + - The subscription server applies it under `_lifecycleGate`, keyed in + `SubscriptionProjectionState` next to `DeliveredBySourceRecordKey` + (`CultNetDatabaseSubscriptionServer.cs:446`). + - `CultNetDatabaseServer.CreateSubscription` (`:501-512`) applies it under a + per-subscription lock. + - `Reconcile` (`:347`) stamps each key it reads with a per-key atomic + `GetWithSequence`. +- **Verification.** Two threads write one key with a blocking observer that + forces the inverted delivery (Q3 shape). The peer's last state for the key + equals the cache's, on both servers. +- **Operator.** Q-P4 asks whether this cut belongs here or is recorded. +- **Ledger.** About +40. + +### Salvage from `hands/cultnet-publish-commits` + +- **Keep:** + - both new test files; + - the `NetworkingTests.cs` edits (`:1528`, `:2257`); + - the door-shape rule; + - "replicas never mint"; + - log-before-publish; + - "the snapshot logs nothing"; + - the rewritten composition-doc paragraph, rewritten again by Cut 3 to name + the journal as the log's order source. +- **Commit** Soul pass 3's P1-P7: P3 in Cut 2, P4 in Cut 1, and the rest in + Cut 4. +- **Drop:** the reorder buffer, the drainer, `_predictionGate`, and logging + from the observer. + +## Subtraction ledger (estimate) + +| Cut | Removed | Added | Other | +|---|---|---|---| +| 1 | ~15 | ~45 | none | +| 2 | ~5 | ~30 | the cache no longer uses R3 `Subject` | +| 3 | 0 | ~50 | one public method (Q-P3) | +| 4 | ~120 vs branch | ~60 | the merge brings +1109/−243, of which about 780 are tests | +| 5 | ~10 | ~30 | the database `Subject` goes, and one public property | +| 6 | 0 | ~40 | none | + +- **Source, net against main:** about +200. +- **Source against the branch:** about −60, and no scheduler anywhere. +- The budget is pressure, not a metric. + +## Operator questions + +- **Q-P1. Order.** + - **A. Order as data** (the `4562340` ruling, kept). + - Every change is published exactly once, on its writer's thread, before + the write returns, and carries `Sequence`. + - There is no cross-thread delivery order. + - The shard log is in cache order because the journal writes it under the + gate. + - Wire projections drop stale changes by Sequence (Cut 6). + - **B. Totally ordered delivery in the cache** (a turnstile, which is Soul's + stated direction). It was built as `bcae483`, deleted for deadlock as + `4562340`, and reproduced by this pass: the networking suite hangs. + Making it safe needs a rule that nobody holding a lock an observer takes + may write the cache, and the subscription server itself breaks that rule. + - **C. A dedicated delivery thread or drainer** (the branch). It was refuted + by Soul pass 3: F3 and F4. + - **Recommended: A.** Only A delivers exactly-once, before-return and + no-deadlock together. The one consumer that needs order gets it where order + is decided. +- **Q-P2. An observer whose exception escapes** (a fail-fast R3 handler, or a + custom observer). + - **A.** Rethrow it to the writer after every observer has run, and keep the + observer subscribed. + - **B.** Report it and unsubscribe. + - **C.** Route it to R3's unhandled handler only. + - **Recommended: A.** + - It matches the `OnUpdate` policy and R3's `OnErrorResume`. + - An escape only happens when the owner asked for fail-fast. + - B turns one bad change into permanent deafness. + - C sends the exception back to the handler that just rethrew it. + - The commit stands under every option. +- **Q-P3. The journal's surface.** + - **A.** A public `IDisposable CultCache.AddJournal(Action>>)`, + guarded so that entering the cache from it throws. + - **B.** An internal method, with `InternalsVisibleTo("GameCult.Networking")`. + The comment at `CultCache.cs:214` shows Networking was deliberately kept + without it. + - **Recommended: A.** + - The contract is small, and the dangerous misuse, reentry, is enforced + rather than documented. + - An ordered admission journal is a legitimate library capability: audit, + replication and derived indexes. + - B couples two assemblies to hide one method. +- **Q-P4. Cut 6 now or later?** + - **A.** Land it in this campaign. + - **B.** Record it. + - **Recommended: A.** With every commit now published, two writers of one key + on two threads can leave a peer holding the older state. That was already + true for loads; commits make it common. + +Not forks, recorded so they are not re-asked: +- A throwing getter refuses loads too. +- Reentrant writes keep main's depth-first delivery. Order is data. +- Journal I/O under the gate follows the store precedent. + +## Follow-ups outside this campaign + +- **`cultmesh-py` watch.** It has no isolation and no Sequence + (`node.py:1059-1065`). +- **`RecordMutationLogEntry` memory-before-disk.** A throwing durable `Append` + leaves memory ahead of disk. This predates the campaign. +- **Durable log whole-file rewrite.** `CultNetShardMutationLogStorage.cs:83-89` + rewrites the whole file per `Append`. It is a monolith, and under the gate + its cost is now every writer's. +- **Mesh database-backed handles.** `CultMesh.cs:2362` and `:3396` can drop + stale changes once `CultNetDatabaseChange.Sequence` exists. +- **Skill scar.** The Eureka SKILL.md already says "a ticket system that + serialized delivery was built, deadlocked, and was deleted". The Soul pass 3 + direction re-proposed it anyway. The Soul brief template should tell Soul to + check `git log -S` for a proposed mechanism before recommending it. From 60663f952cfa8773632c7d8ddd059926294fcfe8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:41:19 +0200 Subject: [PATCH 06/64] Record publication rulings Q-P1..Q-P4 Co-Authored-By: Claude Opus 5.5 --- docs/cultcache-publication-cut.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/cultcache-publication-cut.md b/docs/cultcache-publication-cut.md index f13ba680..7143127a 100644 --- a/docs/cultcache-publication-cut.md +++ b/docs/cultcache-publication-cut.md @@ -15,6 +15,18 @@ exactly once. This document owns the means. - **Scope.** The C# cache's publication and CultNetDatabase's use of it. There is no wire-format change and no other runtime changes (§6). + +## Rulings (operator, 2026-09-30) + +- **Q-P1 A: order as data.** Each change is published exactly once, on its writer's thread, before the write + returns, and carries its `Sequence`. The shard log is written in cache order by the gate-held journal. Wire + peers drop changes that are stale by `Sequence`. The turnstile (`bcae483`/`4562340`) and the drainer are + both rejected. +- **Q-P2 A:** a throwing observer does not stop the others. Every observer runs, the exception is rethrown to + the writer afterwards, and the observer stays subscribed. The commit stands. +- **Q-P3 A:** a public `CultCache.AddJournal(...)`, with reentry guarded. +- **Q-P4 A:** Cut 6, which drops stale changes per key on the wire, lands in this campaign. + ## The finding that shapes this map The direction Imagination was handed was: "ordered delivery belongs in the From 7362257f08a9668bd1bdb6fcd7965ef3d0eebb5c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:48:19 +0200 Subject: [PATCH 07/64] CultCache: read names and indexes once before minting; a throwing getter refuses the whole change set Resolve now ends in Project, the only place an admission runs a [CultName] or [CultIndex] getter on a landing record. It runs before land and before any Sequence is minted, so a throwing getter refuses the admission (loads included) with nothing written, minted or left in memory. The variant name and index rules and the indexer read the projection; Apply runs no user code. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 157 +++++++++---- .../CultCacheProjectionTests.cs | 210 ++++++++++++++++++ .../CultCachePublicationIsolationTests.cs | 63 ++++++ 3 files changed, 388 insertions(+), 42 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs create mode 100644 tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index b2f06659..8af4360c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2465,11 +2465,82 @@ namespace GameCult.Caching // Variants this admission did not write whose base chain it changed: re-resolved, same delta, same storedAt. public List Dependents { get; } = new(); + + // Name and index values of every record that lands (Admitted and Dependents), by key. Filled once by Project. + public Dictionary Projections { get; } = new(StringComparer.Ordinal); + } + + // What a landing record's [CultName] and [CultIndex] getters said. Indexes holds only non-blank values. + private sealed class Projection + { + public Projection(string? name, (string Alias, string Value)[] indexes) + { + Name = name; + Indexes = indexes; + } + + public string? Name { get; } + public (string Alias, string Value)[] Indexes { get; } + } + + // The only place an admission runs a name or index getter on a landing record. It runs before land and before any + // sequence is minted, so a getter that throws refuses the whole admission with nothing written and nothing minted. + private static void Project(VariantPlan plan) + { + foreach (var stored in plan.Admitted.Concat(plan.Dependents)) + { + var descriptor = stored.Descriptor; + string? name = null; + if (descriptor.NameAccessor != null) + name = Read(stored, $"[CultName] member {descriptor.NameMember}", descriptor.NameAccessor) is { Length: > 0 } read ? read : null; + var indexes = new List<(string Alias, string Value)>(); + foreach (var pair in descriptor.IndexAccessors) + { + var member = descriptor.RichMembers.First(candidate => candidate.IndexAlias == pair.Key).Member.Name; + var value = Read(stored, $"index '{pair.Key}' member {member}", pair.Value); + if (!string.IsNullOrWhiteSpace(value)) + indexes.Add((pair.Key, value!)); + } + + plan.Projections[stored.Key.Value] = new Projection(name, indexes.ToArray()); + } + } + + private static string? Read(CultStoredDocument stored, string what, Func getter) + { + try + { + return getter(stored.Document); + } + catch (Exception exception) + { + throw new InvalidOperationException( + $"Record {stored.Key.Value} ({stored.Descriptor.SchemaName}) is refused: its {what} threw: {exception.GetBaseException().Message}", exception); + } + } + + // Every admission is judged here: variants resolved, then every landing record's name and index values read once, + // then the rules that compare them. It refuses naming the keys, and mutates nothing. + private VariantPlan Resolve( + IReadOnlyList admitted, + IReadOnlyList evicted, + CacheBackingStore? source) + { + var plan = ResolveVariants(admitted, evicted, source); + Project(plan); + // The variant rules are judged only when a variant is involved. + if (_variantKeys.Count != 0 || admitted.Any(stored => stored.Variant != null)) + { + RefuseVariantNameSharing(plan, evicted); + RefuseVariantIndexSharing(plan, evicted); + } + + return plan; } // Resolves every admitted variant and every variant whose base chain this admission changes, base first, against // the post-admission set. It refuses a broken variant naming the keys, and mutates nothing. - private VariantPlan Resolve( + private VariantPlan ResolveVariants( IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source) @@ -2531,11 +2602,7 @@ namespace GameCult.Caching } if (toResolve.Count == 0) - { - var unchanged = new VariantPlan(admitted); - RefuseVariantIndexSharing(unchanged, incoming, gone); - return unchanged; - } + return new VariantPlan(admitted); var pending = new HashSet(toResolve, StringComparer.Ordinal); var done = new Dictionary(StringComparer.Ordinal); @@ -2546,7 +2613,6 @@ namespace GameCult.Caching var plan = new VariantPlan(admitted.Select(stored => done.TryGetValue(stored.Key.Value, out var resolved) ? resolved : stored).ToList()); foreach (var key in toResolve.Where(key => !incoming.ContainsKey(key))) plan.Dependents.Add(done[key]); - RefuseVariantIndexSharing(plan, incoming, gone); return plan; CultStoredDocument ResolveOne(string key) @@ -2610,21 +2676,6 @@ namespace GameCult.Caching var document = _codec.Deserialize(type, type, kept.Count == 0 ? payload : _codec.Overlay(payload, kept)) ?? throw new InvalidOperationException($"Variant {key} resolved to nothing."); - if (isVariant) - { - // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. - if (descriptor.NameAccessor?.Invoke(document) is { Length: > 0 } variantName) - { - for (var ancestor = baseStored; ancestor != null; - ancestor = ancestor.Variant == null ? null : done.TryGetValue(ancestor.Variant.BaseKey, out var above) ? above : Lookup(ancestor.Variant.BaseKey)) - { - if (descriptor.NameAccessor(ancestor.Document) == variantName) - throw new InvalidOperationException( - $"Variant {key} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); - } - } - } - stack.RemoveAt(stack.Count - 1); return done[key] = isVariant ? variant.Resolved(document) @@ -2632,20 +2683,49 @@ namespace GameCult.Caching } } + // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. A variant's + // name is its projection; an ancestor that lands is read from its projection, one that stays is read as held. + private void RefuseVariantNameSharing(VariantPlan plan, IReadOnlyList evicted) + { + var gone = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); + var landing = plan.Admitted.Concat(plan.Dependents).ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); + + CultStoredDocument? Ancestor(string key) => + landing.TryGetValue(key, out var landed) ? landed + : gone.Contains(key) ? null + : _entries.TryGetValue(key, out var held) ? held : null; + + string? NameOf(CultStoredDocument stored) => + plan.Projections.TryGetValue(stored.Key.Value, out var projection) + ? projection.Name + : stored.Descriptor.NameAccessor == null ? null : Read(stored, $"[CultName] member {stored.Descriptor.NameMember}", stored.Descriptor.NameAccessor); + + foreach (var variant in landing.Values.Where(stored => stored.Variant != null)) + { + if (plan.Projections[variant.Key.Value].Name is not { } variantName) + continue; + for (var ancestor = Ancestor(variant.Variant!.BaseKey); ancestor != null; + ancestor = ancestor.Variant == null ? null : Ancestor(ancestor.Variant.BaseKey)) + { + if (NameOf(ancestor) == variantName) + throw new InvalidOperationException( + $"Variant {variant.Key.Value} and its base {ancestor.Key.Value} are both named '{variantName}'; a variant must override its name."); + } + } + } + // The one index rule, judged on the commit's result whatever the write order: a variant may not share an indexed // value with another record. It reads the truthful held index (every holder of a value), minus the records this // commit replaces or removes, plus the records it lands. Plain-vs-plain sharing is not this rule's business. // A unique index maps a value to one key at lookup; a variant sharing it would be found in place of its // holder or lose the value, and the store would refuse to reopen. - private void RefuseVariantIndexSharing( - VariantPlan plan, - IReadOnlyDictionary incoming, - HashSet gone) + private void RefuseVariantIndexSharing(VariantPlan plan, IReadOnlyList evicted) { var landing = plan.Admitted.Concat(plan.Dependents).ToList(); if (landing.All(stored => stored.Descriptor.IndexAccessors.Count == 0)) return; - var replaced = new HashSet(incoming.Keys.Concat(gone).Concat(plan.Dependents.Select(stored => stored.Key.Value)), StringComparer.Ordinal); + var replaced = new HashSet( + plan.Admitted.Concat(evicted).Concat(plan.Dependents).Select(stored => stored.Key.Value), StringComparer.Ordinal); var landingByKey = landing.ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); bool IsVariant(string key) => @@ -2656,12 +2736,9 @@ namespace GameCult.Caching var landed = new Dictionary<((Type Type, string Alias) Index, string Value), List>(); foreach (var stored in landing) { - foreach (var pair in stored.Descriptor.IndexAccessors) + foreach (var (alias, value) in plan.Projections[stored.Key.Value].Indexes) { - var value = pair.Value(stored.Document); - if (string.IsNullOrWhiteSpace(value)) - continue; - var index = (stored.Descriptor.DocumentType, pair.Key); + var index = (stored.Descriptor.DocumentType, alias); landedValues.Add((stored, index, value)); if (!landed.TryGetValue((index, value), out var landedKeys)) landed[(index, value)] = landedKeys = new List(); @@ -2733,7 +2810,7 @@ namespace GameCult.Caching _entries[stored.Key.Value] = stored; _handles.AddOrUpdate(stored.Document, new KeyBox(stored.Key)); - Index(stored); + Index(stored, plan.Projections[stored.Key.Value]); if (stored.Variant == null) _variantKeys.Remove(stored.Key.Value); else @@ -2790,19 +2867,15 @@ namespace GameCult.Caching } } - private void Index(CultStoredDocument stored) + private void Index(CultStoredDocument stored, Projection projection) { var type = stored.Descriptor.DocumentType; if (stored.Descriptor.IsGlobal) _globals[type] = stored.Key.Value; - if (stored.Descriptor.NameAccessor?.Invoke(stored.Document) is { Length: > 0 } name) - MapOf(_names, type)[name] = stored.Key.Value; - foreach (var pair in stored.Descriptor.IndexAccessors) - { - var value = pair.Value(stored.Document); - if (!string.IsNullOrWhiteSpace(value)) - AddHolder(_indexes, (type, pair.Key), value, stored.Key.Value); - } + if (projection.Name != null) + MapOf(_names, type)[projection.Name] = stored.Key.Value; + foreach (var (alias, value) in projection.Indexes) + AddHolder(_indexes, (type, alias), value, stored.Key.Value); } // Documents are mutable, so a name or index value may have changed since it was indexed: drop by key. diff --git a/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs new file mode 100644 index 00000000..a35db5bd --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs @@ -0,0 +1,210 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // Cut 1 of docs/cultcache-publication-cut.md: an admission reads every landing record's name and index values once, + // before land and before any sequence is minted. A getter that throws refuses the whole change set. + [NonParallelizable] + public class CultCacheProjectionTests + { + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(ProjectedRecord) }); + + private string _directory = string.Empty; + + [SetUp] + public void Prepare() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-projection-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + ProjectedRecord.Armed = false; + ProjectedRecord.Reads = 0; + } + + [TearDown] + public void Cleanup() + { + ProjectedRecord.Armed = false; + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + [CultDocument("tests.projected_record", "tests.projected_record.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class ProjectedRecord + { + public static long Reads; + public static volatile bool Armed; + [IgnoreMember] private string _name = string.Empty; + [IgnoreMember] private string _code = string.Empty; + + [Key(0)] + [CultName] + public string Name + { + get + { + Interlocked.Increment(ref Reads); + if (Armed && _name == "boom") + throw new InvalidOperationException("name getter boom"); + return _name; + } + set => _name = value; + } + + [Key(1)] + [CultIndex("code")] + public string Code + { + get + { + Interlocked.Increment(ref Reads); + if (Armed && _code == "boom") + throw new InvalidOperationException("index getter boom"); + return _code; + } + set => _code = value; + } + } + + private static readonly CultRecordKey OkKey = new("projected:ok"); + private static readonly CultRecordKey BoomKey = new("projected:boom"); + + private static ProjectedRecord Ok() => new() { Name = "ok", Code = "ok-code" }; + + private static ProjectedRecord Boom(bool name) => name ? new ProjectedRecord { Name = "boom", Code = "boom-code" } : new ProjectedRecord { Name = "boom-name", Code = "boom" }; + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path) => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + + private static void Put(CultCache cache, ProjectedRecord document, CultRecordKey key) => + cache.Commit(batch => batch.Upsert(typeof(ProjectedRecord), document, key)); + + private static long SequenceOf(CultCache cache) => cache.GetWithSequence(OkKey).Sequence; + + private static void StageBoth(CultCacheBatch batch, ProjectedRecord boom) + { + batch.Upsert(typeof(ProjectedRecord), Ok(), OkKey); + batch.Upsert(typeof(ProjectedRecord), boom, BoomKey); + } + + [TestCase(true, "[CultName] member Name")] + [TestCase(false, "index 'code' member Code")] + public void ARefusedGetterMintsNothingAndChangesNothing(bool nameThrows, string member) + { + using var cache = new CultCache(Registry); + var seen = new List(); + using var watch = cache.Watch().Subscribe(change => seen.Add(change.Sequence)); + Put(cache, new ProjectedRecord { Name = "seed", Code = "seed-code" }, new CultRecordKey("projected:seed")); + var before = SequenceOf(cache); + seen.Clear(); + ProjectedRecord.Armed = true; + + var refusal = Assert.Throws(() => cache.Commit(batch => StageBoth(batch, Boom(nameThrows))))!; + + ProjectedRecord.Armed = false; + Assert.Multiple(() => + { + Assert.That(refusal.Message, Does.Contain(BoomKey.Value).And.Contain(member).And.Contain("getter boom")); + Assert.That(SequenceOf(cache), Is.EqualTo(before), "a refused admission mints nothing"); + Assert.That(cache.Get(OkKey), Is.Null, "the good record did not land"); + Assert.That(cache.Get(BoomKey), Is.Null); + Assert.That(cache.GetByName("ok"), Is.Null, "the name index holds nothing of the refused set"); + Assert.That(cache.GetByIndex("code", "ok-code"), Is.Null); + Assert.That(seen, Is.Empty, "nothing was published"); + }); + + Put(cache, Ok(), OkKey); + Assert.That(SequenceOf(cache), Is.EqualTo(before + 1), "the next write is previous + 1"); + Assert.That(seen, Is.EqualTo(new[] { before + 1 })); + } + + [Test] + public void ARefusedSingleWriteMintsNothing() + { + using var cache = new CultCache(Registry); + ProjectedRecord.Armed = true; + + var refusal = Assert.Throws(() => Put(cache, Boom(true), BoomKey))!; + + ProjectedRecord.Armed = false; + Assert.That(refusal.Message, Does.Contain(BoomKey.Value)); + Assert.That(SequenceOf(cache), Is.EqualTo(0)); + Assert.That(cache.Get(BoomKey), Is.Null); + } + + [Test] + public void ARefusedBatchLeavesAStoreByteIdenticalAndReopensWithNeitherKey() + { + var path = PathOf("projected.cc"); + using (var seed = Open(path)) + Put(seed, new ProjectedRecord { Name = "seed", Code = "seed-code" }, new CultRecordKey("projected:seed")); + var before = File.ReadAllBytes(path); + + using (var cache = Open(path)) + { + ProjectedRecord.Armed = true; + Assert.Throws(() => cache.Commit(batch => StageBoth(batch, Boom(true)))); + ProjectedRecord.Armed = false; + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + } + + using var reopened = Open(path); + Assert.Multiple(() => + { + Assert.That(reopened.Get(OkKey), Is.Null); + Assert.That(reopened.Get(BoomKey), Is.Null); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + }); + } + + [Test] + public void ALoadWhoseGetterThrowsRefusesTheOpenAndLeavesTheFileByteIdentical() + { + var path = PathOf("poisoned.cc"); + using (var seed = Open(path)) + Put(seed, Boom(true), BoomKey); + var before = File.ReadAllBytes(path); + + ProjectedRecord.Armed = true; + var refusal = Assert.Throws(() => Open(path))!; + ProjectedRecord.Armed = false; + + Assert.That(refusal.Message, Does.Contain(BoomKey.Value).And.Contain("[CultName] member Name")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + } + + // A counting getter: each landing record is read once for its name and once for its index, and never again by the + // variant rules or the indexer. A variant re-resolved by a base edit lands too, and is read once. + [Test] + public void EachLandedRecordIsReadOncePerGetterIncludingAVariantDependent() + { + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + var baseKey = new CultRecordKey("projected:base"); + Put(cache, new ProjectedRecord { Name = "base", Code = "base-code" }, baseKey); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("projected:variant"), baseKey, new[] + { + cache.Override(nameof(ProjectedRecord.Name), "variant"), + cache.Override(nameof(ProjectedRecord.Code), "variant-code") + })); + + ProjectedRecord.Reads = 0; + Put(cache, new ProjectedRecord { Name = "base", Code = "base-code-2" }, baseKey); + + // Two landing records (the base, and the variant re-resolved against it), two getters each. + Assert.That(Interlocked.Read(ref ProjectedRecord.Reads), Is.EqualTo(4)); + Assert.That(cache.GetByIndex("code", "base-code-2"), Is.Not.Null); + Assert.That(cache.GetByName("variant"), Is.Not.Null); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs new file mode 100644 index 00000000..9ba78efa --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs @@ -0,0 +1,63 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + [CultDocument("tests.publication_throwing_name", "tests.publication_throwing_name.v1")] + [MessagePack.MessagePackObject] + public sealed class PublicationThrowingName + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultName] + public string Name + { + get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text; + set { } + } + } + + // Cache publication cuts 1 and 2 seen through CultNetDatabase: a refused or failing change never stalls the + // publication and log of the changes after it. (Soul pass 3 probes P4 and P3.) + [NonParallelizable] + public sealed class CultCachePublicationIsolationTests + { + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId => CultDocumentRegistry.Shared.GetRequired().SchemaId; + + // An admission that a [CultName] getter refuses publishes nothing, leaves nothing in memory, and stalls nothing after it. + [Test] + public async Task ARefusedAdmissionLeavesNothingAndLaterChangesStillPublishAndLog() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + var okKey = new CultRecordKey("pub:ok"); + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(new PublicationThrowingName { Text = "ok" }, new CultRecordHandle(okKey)); + batch.Upsert(new PublicationThrowingName { Text = "boom" }, new CultRecordHandle(new CultRecordKey("pub:boomname"))); + }))!; + + var after = new CultRecordKey("pub:after"); + await database.PutAsync(after, Note("after")); + Assert.Multiple(() => + { + Assert.That(refusal.Message, Does.Contain("pub:boomname").And.Contain("name getter boom")); + Assert.That(cache.Get(okKey), Is.Null, "the good record of a refused batch did not land"); + Assert.That(seen, Is.EqualTo(new[] { "after" })); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + } +} From 6b0cf66f0e1e2066b2867ccd1a898a4f0a19b716 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:50:29 +0200 Subject: [PATCH 08/64] Count serializer reads apart in the once-per-landed-record projection test Resolving a variant serializes its base, and the serializer reads the same getters; the test measured them together and expected too few. Co-Authored-By: Claude Sonnet 5 --- .../CultCacheProjectionTests.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs index a35db5bd..88f251c6 100644 --- a/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs @@ -189,7 +189,8 @@ namespace GameCult.Caching.Tests [Test] public void EachLandedRecordIsReadOncePerGetterIncludingAVariantDependent() { - using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + var codec = CultCacheMessagePack.CreateCodec(Registry); + using var cache = new CultCache(Registry, codec); var baseKey = new CultRecordKey("projected:base"); Put(cache, new ProjectedRecord { Name = "base", Code = "base-code" }, baseKey); cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("projected:variant"), baseKey, new[] @@ -198,11 +199,16 @@ namespace GameCult.Caching.Tests cache.Override(nameof(ProjectedRecord.Code), "variant-code") })); + // Resolving a variant serializes its base, and the serializer reads the same getters: count those reads apart. + ProjectedRecord.Reads = 0; + codec.Serialize(new ProjectedRecord(), typeof(ProjectedRecord), typeof(ProjectedRecord)); + var serializerReads = Interlocked.Read(ref ProjectedRecord.Reads); + ProjectedRecord.Reads = 0; Put(cache, new ProjectedRecord { Name = "base", Code = "base-code-2" }, baseKey); - // Two landing records (the base, and the variant re-resolved against it), two getters each. - Assert.That(Interlocked.Read(ref ProjectedRecord.Reads), Is.EqualTo(4)); + // Two landing records (the base, and the variant re-resolved against it), two getters each, plus the one serialization. + Assert.That(Interlocked.Read(ref ProjectedRecord.Reads), Is.EqualTo(4 + serializerReads)); Assert.That(cache.GetByIndex("code", "base-code-2"), Is.Not.Null); Assert.That(cache.GetByName("variant"), Is.Not.Null); } From ecee41e8fff510b3616d44e7582b6e34ee038036 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:51:33 +0200 Subject: [PATCH 09/64] CultCache: own the observer list and isolate every observer The cache no longer fans changes out through an R3 Subject. It keeps a copy-on-write observer array and Publish catches per observer: every observer receives every change, an exception that escapes (a fail-fast R3 handler) joins the OnUpdate failures rethrown to the writer after delivery, and the observer stays subscribed. Dispose completes observers, and a disposed cache completes new ones at once. Deletes the false "OnNext does not throw" comment. The composition contract now states the observer policy and that a throwing name or index getter refuses a change set, loads included. Co-Authored-By: Claude Sonnet 5 --- .../Contracts/cultcache-store-composition.md | 17 ++- src/GameCult.Caching/CultCache.cs | 73 ++++++++- .../CultCacheObserverTests.cs | 139 ++++++++++++++++++ .../CultCachePublicationIsolationTests.cs | 35 +++++ 4 files changed, 252 insertions(+), 12 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/CultCacheObserverTests.cs diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index ab028fca..6cf6cdfa 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -77,14 +77,21 @@ which route wrote it. - An `OnUpdate` handler exception is rethrown to that caller after all of that call's changes are delivered (an `AggregateException` if several threw). If the call itself failed, its own exception is rethrown and handler exceptions - are dropped. A `Watch` subscriber exception is not rethrown; it follows R3's - unhandled-exception handling. A throwing handler cannot undo the store's - adoption of a load; the store and cache already agree when publication starts. + are dropped. Every `Watch` subscriber receives every change: R3 routes a + subscriber's exception to its unhandled-exception handler, and if that handler + rethrows (fail-fast), the exception is treated as an `OnUpdate` handler + exception. The subscriber stays subscribed and the commit stands. A throwing + handler cannot undo the store's adoption of a load; the store and cache + already agree when publication starts. +- A record's `[CultName]` and `[CultIndex]` getters run once per landing record, + before the store is written and before any `Sequence` is minted. A getter that + throws refuses the whole change set, loads included: nothing is written, + minted or left in memory. - Pulling all stores pulls every attached store even if a handler throws during one store's load, then rethrows. - Hydration failure on open is loud: a corrupt store file, or a record whose - schema the registry cannot resolve, makes the open throw and leaves the file - byte-identical. Consumers never delete and rewrite a store they failed to + schema the registry cannot resolve or whose name or index getter throws, makes + the open throw and leaves the file byte-identical. Consumers never delete and rewrite a store they failed to open. ## Locking diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 8af4360c..2a9de06e 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1806,7 +1806,11 @@ namespace GameCult.Caching // Keys holding a variant: derived from the entries by Apply, never decided anywhere else. private readonly HashSet _variantKeys = new(StringComparer.Ordinal); private readonly ConditionalWeakTable _handles = new(); - private readonly Subject _changes = new(); + // The cache's own observers, copy-on-write under _observerGate: publication reads one snapshot per change and + // catches per observer, so no observer decides whether another receives a change. + private Observer[] _observers = Array.Empty>(); + private bool _observersCompleted; + private readonly object _observerGate = new(); private readonly object _gate = new(); private long _sequence; // The changes admitted by this thread's outermost hold, published by that hold when it exits. @@ -1886,7 +1890,7 @@ namespace GameCult.Caching public Observable> Watch() where T : class { - return _changes + return Observable.Create(Register) .Where(change => typeof(T).IsAssignableFrom(change.Stored.Descriptor.DocumentType)) .Select(change => new CultCacheDocumentChange( change.Kind, @@ -1896,6 +1900,25 @@ namespace GameCult.Caching change.Sequence)); } + private IDisposable Register(Observer observer) + { + lock (_observerGate) + { + if (!_observersCompleted) + { + _observers = _observers.Append(observer).ToArray(); + return Disposable.Create(() => + { + lock (_observerGate) + _observers = _observers.Where(held => !ReferenceEquals(held, observer)).ToArray(); + }); + } + } + + observer.OnCompleted(); + return Disposable.Empty; + } + public Observable> WatchRecord(CultRecordKey key) where T : class { return Watch().Where(change => change.Key.Equals(key)); @@ -2151,7 +2174,31 @@ namespace GameCult.Caching foreach (var store in BackingStores) store.Dispose(); - _changes.Dispose(); + Observer[] completing; + lock (_observerGate) + { + completing = _observers; + _observers = Array.Empty>(); + _observersCompleted = true; + } + + var failures = new List(); + foreach (var observer in completing) + { + try + { + observer.OnCompleted(); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + if (failures.Count == 1) + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + if (failures.Count > 1) + throw new AggregateException(failures); } internal static void RequireInstanceOf(Type documentType, object document) @@ -2333,8 +2380,8 @@ namespace GameCult.Caching // Every admission runs in a hold. A nested hold on the same cache adds to the outermost one, which publishes // exactly its own changes after it leaves the gate, before it returns, on its caller's thread: observers never // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not - // guaranteed; each change carries the Sequence it was admitted with. Every change is delivered even if an - // OnUpdate handler throws; then the first handler exception (an AggregateException for several) is rethrown. + // guaranteed; each change carries the Sequence it was admitted with. Every change reaches every observer even + // if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown. // If the body threw, its exception wins. internal T Held(Func body) { @@ -2371,8 +2418,20 @@ namespace GameCult.Caching var failures = new List(); foreach (var (change, loaded) in changes) { - // R3 routes a throwing subscriber to its unhandled-exception handler; OnNext does not throw. - _changes.OnNext(change); + // R3 routes a throwing subscriber to its unhandled-exception handler, and a fail-fast handler rethrows: every + // observer still runs, the exception joins the failures, and the observer stays subscribed. + foreach (var observer in Volatile.Read(ref _observers)) + { + try + { + observer.OnNext(change); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + if (!loaded) continue; try diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs new file mode 100644 index 00000000..8cb3992d --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -0,0 +1,139 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // Cut 2 of docs/cultcache-publication-cut.md: the cache keeps its own observers and catches per observer, so no + // observer decides whether another receives a change. R3 routes a subscriber's exception to the unhandled handler; + // these tests install a fail-fast (rethrowing) handler, the only way an exception reaches the cache. + [NonParallelizable] + public class CultCacheObserverTests + { + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(ObservedPing) }); + + private Action _previousHandler = _ => { }; + + [SetUp] + public void FailFast() + { + _previousHandler = ObservableSystem.GetUnhandledExceptionHandler(); + ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); + } + + [TearDown] + public void Restore() => ObservableSystem.RegisterUnhandledExceptionHandler(_previousHandler); + + [CultDocument("tests.observed_ping", "tests.observed_ping.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class ObservedPing + { + [Key(0)] + public string Text = string.Empty; + } + + private static CultRecordKey KeyOf(string text) => new($"observed:{text}"); + + private static void Send(CultCache cache, params string[] texts) => + cache.Commit(batch => + { + foreach (var text in texts) + batch.Upsert(typeof(ObservedPing), new ObservedPing { Text = text }, KeyOf(text)); + }); + + [Test] + public void AnEscapedObserverExceptionReachesTheWriterAfterEveryObserverSawEveryChange() + { + using var cache = new CultCache(Registry); + var thrower = new List(); + var later = new List(); + using var first = cache.Watch().Subscribe(change => + { + thrower.Add(change.Document!.Text); + if (change.Document.Text == "boom") + throw new InvalidOperationException("observer boom"); + }); + using var second = cache.Watch().Subscribe(change => later.Add(change.Document!.Text)); + + var escaped = Assert.Throws(() => Send(cache, "boom", "after"))!; + + Assert.Multiple(() => + { + Assert.That(escaped.Message, Is.EqualTo("observer boom")); + Assert.That(later, Is.EqualTo(new[] { "boom", "after" }), "the observer after the thrower saw every change of the hold"); + Assert.That(thrower, Is.EqualTo(new[] { "boom", "after" }), "the thrower kept receiving after its own exception"); + Assert.That(cache.Get(KeyOf("boom")), Is.Not.Null, "the commit stands"); + Assert.That(cache.Get(KeyOf("after")), Is.Not.Null); + }); + + Send(cache, "later"); + Assert.That(thrower, Is.EqualTo(new[] { "boom", "after", "later" }), "the thrower stayed subscribed"); + Assert.That(later, Is.EqualTo(new[] { "boom", "after", "later" })); + } + + [Test] + public void SeveralEscapedObserverExceptionsArriveTogether() + { + using var cache = new CultCache(Registry); + using var one = cache.Watch().Subscribe(_ => throw new InvalidOperationException("one")); + using var two = cache.Watch().Subscribe(_ => throw new InvalidOperationException("two")); + + var escaped = Assert.Throws(() => Send(cache, "boom"))!; + + Assert.That(escaped.InnerExceptions.Select(exception => exception.Message), Is.EqualTo(new[] { "one", "two" })); + } + + [Test] + public void ObserversMayJoinAndLeaveDuringADeliveryWithoutBreakingTheOthers() + { + using var cache = new CultCache(Registry); + var a = new List(); + var b = new List(); + var c = new List(); + var d = new List(); + IDisposable? leaving = null; + IDisposable? joined = null; + using var first = cache.Watch().Subscribe(change => + { + a.Add(change.Sequence); + if (a.Count != 1) + return; + leaving!.Dispose(); + joined = cache.Watch().Subscribe(next => c.Add(next.Sequence)); + }); + leaving = cache.Watch().Subscribe(change => b.Add(change.Sequence)); + using var last = cache.Watch().Subscribe(change => d.Add(change.Sequence)); + + Send(cache, "one", "two"); + joined?.Dispose(); + + Assert.Multiple(() => + { + Assert.That(a, Is.EqualTo(new[] { 1L, 2L })); + Assert.That(d, Is.EqualTo(new[] { 1L, 2L }), "an observer that did not move still saw everything"); + Assert.That(b, Does.Not.Contain(2L), "a disposed observer stops receiving"); + Assert.That(c, Is.EqualTo(new[] { 2L }), "an observer that joined mid-hold receives from the next change"); + }); + } + + [Test] + public void DisposeCompletesObserversAndACompletedCacheCompletesNewOnes() + { + var cache = new CultCache(Registry); + var completed = 0; + using var before = cache.Watch().Subscribe(_ => { }, _ => completed++); + + cache.Dispose(); + Assert.That(completed, Is.EqualTo(1)); + + using var after = cache.Watch().Subscribe(_ => { }, _ => completed++); + Assert.That(completed, Is.EqualTo(2), "subscribing to a disposed cache completes at once"); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs index 9ba78efa..33d1100d 100644 --- a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs +++ b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs @@ -59,5 +59,40 @@ namespace GameCult.Networking.Tests Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); }); } + + // One observer's escaped exception (a fail-fast R3 handler) does not stop the database observer after it, and the writer gets it. + [Test] + public async Task AnEscapedObserverExceptionDoesNotStopTheDatabaseOrLaterChanges() + { + var previous = ObservableSystem.GetUnhandledExceptionHandler(); + ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); + try + { + var cache = new CultCache(); + using var thrower = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom"); + }); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + var boom = new CultRecordKey("pub:boom"); + var escaped = Assert.ThrowsAsync(() => cache.UpsertAsync(Note("boom"), new CultRecordHandle(boom))); + + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + var after = new CultRecordKey("pub:after"); + await database.PutAsync(after, Note("after")); + Assert.Multiple(() => + { + Assert.That(escaped!.Message, Is.EqualTo("observer boom")); + Assert.That(seen, Is.EqualTo(new[] { "boom", "after" }), "the database observer after the thrower saw both changes"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + finally + { + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + } + } } } From 5687d8c3558495ac285126e7af90204c859fddae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:52:44 +0200 Subject: [PATCH 10/64] Make the CultLib Unity package depend on org.gamecult.cultmath GameCult.Caching.MessagePack now references CultMath, so the Unity package declares org.gamecult.cultmath in package.json, references CultMath.dll from the asmdef, and the package build refuses to ship a second CultMath.dll and checks the asmdef against the shipped plus external assembly lists. Co-Authored-By: Claude Sonnet 5 --- scripts/build-unity-package.ps1 | 18 ++++++++++++++++++ unity/org.gamecult.cultlib/README.md | 9 +++++++++ .../Runtime/GameCult.CultLib.asmdef | 1 + unity/org.gamecult.cultlib/package.json | 3 +++ 4 files changed, 31 insertions(+) diff --git a/scripts/build-unity-package.ps1 b/scripts/build-unity-package.ps1 index bdd9fff6..3258155f 100644 --- a/scripts/build-unity-package.ps1 +++ b/scripts/build-unity-package.ps1 @@ -128,6 +128,19 @@ $expectedAssemblies = @( "System.Text.Json.dll", "System.Threading.Channels.dll" ) +# Assemblies the package references but never ships: org.gamecult.cultmath owns CultMath.dll, and the +# package.json dependency delivers it. A second copy in Runtime\Plugins would be a duplicate assembly. +$externalAssemblies = @("CultMath.dll") +$asmdef = Get-Content -LiteralPath (Join-Path $templateRoot "Runtime\GameCult.CultLib.asmdef") -Raw | ConvertFrom-Json +$referencedAssemblies = @($asmdef.precompiledReferences | Sort-Object) +$declaredAssemblies = @($expectedAssemblies + $externalAssemblies | Sort-Object) +if (($referencedAssemblies -join "|") -ne ($declaredAssemblies -join "|")) { + throw "GameCult.CultLib.asmdef precompiledReferences differ from the shipped plus external assembly lists." +} +$cultMathDependency = (Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json).dependencies.'org.gamecult.cultmath' +if ([string]::IsNullOrWhiteSpace($cultMathDependency)) { + throw "package.json must declare org.gamecult.cultmath, which supplies CultMath.dll." +} $publishedByName = @{} foreach ($assembly in Get-ChildItem -LiteralPath $publishRoot -Filter "*.dll") { $publishedByName[$assembly.Name] = $assembly @@ -154,6 +167,11 @@ foreach ($assemblyName in $expectedAssemblies) { Copy-Item -LiteralPath $pdb -Destination $pluginRoot } } +foreach ($external in $externalAssemblies) { + if (Test-Path -LiteralPath (Join-Path $pluginRoot $external)) { + throw "CultLib Unity package must not ship $external; org.gamecult.cultmath owns it." + } +} $nativePluginRoot = Join-Path $pluginRoot "x86_64" New-Item -ItemType Directory -Force -Path $nativePluginRoot | Out-Null Copy-Item -LiteralPath (Join-Path $quicNativeRoot "gamecult_mesh_quic_native.dll") -Destination $nativePluginRoot diff --git a/unity/org.gamecult.cultlib/README.md b/unity/org.gamecult.cultlib/README.md index 88a083e1..8716d7e4 100644 --- a/unity/org.gamecult.cultlib/README.md +++ b/unity/org.gamecult.cultlib/README.md @@ -17,6 +17,15 @@ Install version `1.0.57` through Unity Package Manager with: https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v1.0.57 ``` +The package depends on `org.gamecult.cultmath` (0.3.0 or later) and does not +carry its own `CultMath.dll`. Unity Package Manager resolves git packages only +from the project manifest, so add CultMath beside CultLib in +`Packages/manifest.json`: + +```text +"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath#cultmath-unity-v0.3.0" +``` + CultMesh chooses mapped memory for reachable same-machine bodies and advertised QUIC for remote realtime state. Applications register the packaged `CultMeshNativeQuicRealtimeTransportConnector`; they do not load or configure diff --git a/unity/org.gamecult.cultlib/Runtime/GameCult.CultLib.asmdef b/unity/org.gamecult.cultlib/Runtime/GameCult.CultLib.asmdef index bd13cb38..0ddbb9a6 100644 --- a/unity/org.gamecult.cultlib/Runtime/GameCult.CultLib.asmdef +++ b/unity/org.gamecult.cultlib/Runtime/GameCult.CultLib.asmdef @@ -8,6 +8,7 @@ "overrideReferences": true, "precompiledReferences": [ "ConcurrentCollections.dll", + "CultMath.dll", "GameCult.Caching.dll", "GameCult.Caching.MessagePack.dll", "GameCult.Logging.dll", diff --git a/unity/org.gamecult.cultlib/package.json b/unity/org.gamecult.cultlib/package.json index 722190cf..6545bed1 100644 --- a/unity/org.gamecult.cultlib/package.json +++ b/unity/org.gamecult.cultlib/package.json @@ -3,6 +3,9 @@ "displayName": "CultLib", "version": "1.0.60", "unity": "2021.3", + "dependencies": { + "org.gamecult.cultmath": "0.3.0" + }, "description": "CultCache, CultNet, and CultMesh managed runtime assemblies for Unity.", "license": "MIT", "author": { From b8ceab84acdbc80a3f1babb4075070e53d5cc029 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:54:40 +0200 Subject: [PATCH 11/64] Drive the escaped-observer probe through a database write A direct cache write is not a database change on this base, so the database observer could not show it. PutAsync is, and the observer after the thrower must see it. Co-Authored-By: Claude Sonnet 5 --- .../CultCachePublicationIsolationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs index 33d1100d..8b6106ec 100644 --- a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs +++ b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs @@ -77,7 +77,7 @@ namespace GameCult.Networking.Tests var seen = new ConcurrentQueue(); using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); var boom = new CultRecordKey("pub:boom"); - var escaped = Assert.ThrowsAsync(() => cache.UpsertAsync(Note("boom"), new CultRecordHandle(boom))); + var escaped = Assert.ThrowsAsync(() => database.PutAsync(boom, Note("boom"))); ObservableSystem.RegisterUnhandledExceptionHandler(previous); var after = new CultRecordKey("pub:after"); From 41b964c6f1cc3c8df48f71b0617ad29dae257e58 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:57:04 +0200 Subject: [PATCH 12/64] Narrow the escaped-observer probe to what this base can show A database write is published after its cache write returns, so a write whose observer threw never reaches the database observers on this base. Keep the probe to the exception reaching the writer and later changes still publishing and logging; the ordering question belongs to the database cut. Co-Authored-By: Claude Sonnet 5 --- .../CultCachePublicationIsolationTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs index 8b6106ec..f8eeec8f 100644 --- a/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs +++ b/tests/GameCult.Networking.Tests/CultCachePublicationIsolationTests.cs @@ -60,9 +60,11 @@ namespace GameCult.Networking.Tests }); } - // One observer's escaped exception (a fail-fast R3 handler) does not stop the database observer after it, and the writer gets it. + // One observer's escaped exception (a fail-fast R3 handler) reaches the writer and does not stall later changes. + // A database write is published by the database after its cache write returns, so the failed write itself is not + // published here; that ordering belongs to the CultNetDatabase cut. [Test] - public async Task AnEscapedObserverExceptionDoesNotStopTheDatabaseOrLaterChanges() + public async Task AnEscapedObserverExceptionReachesTheWriterAndDoesNotStallLaterChanges() { var previous = ObservableSystem.GetUnhandledExceptionHandler(); ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); @@ -85,7 +87,7 @@ namespace GameCult.Networking.Tests Assert.Multiple(() => { Assert.That(escaped!.Message, Is.EqualTo("observer boom")); - Assert.That(seen, Is.EqualTo(new[] { "boom", "after" }), "the database observer after the thrower saw both changes"); + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); }); } From 77602d5c0eb01b2e34bf1bb9820432e77c78c50a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:57:31 +0200 Subject: [PATCH 13/64] Pin every CultMath MessagePack shape and decode a missing Color32 alpha as opaque Golden bytes now cover double2, double3, int3, int4, bool3, bool4, quaternion, Color32, Random, float3x3 and CultCellular. A missing Color32 alpha decodes as 255, the constructor default. A test proves a consumer resolver registered for the assembly wins over CultMathResolver. The redundant nil check is deleted; ReadArrayHeader already refuses nil. Co-Authored-By: Claude Sonnet 5 --- .../CultMathResolver.cs | 5 +- .../CultMathSerializationTests.cs | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultMathResolver.cs b/src/GameCult.Caching.MessagePack/CultMathResolver.cs index c854e12d..803b3873 100644 --- a/src/GameCult.Caching.MessagePack/CultMathResolver.cs +++ b/src/GameCult.Caching.MessagePack/CultMathResolver.cs @@ -11,7 +11,7 @@ namespace GameCult.Caching.MessagePack; /// The canonical MessagePack encoding of every public CultMath value type. Each type is an /// array of its components in declaration order: vectors and quaternions are flat scalar /// arrays, matrices are arrays of row vectors, and rect is [min, max]. Decoding -/// skips trailing elements and leaves missing ones at zero; nil is refused. The resolver is +/// skips trailing elements and leaves missing ones at zero (a missing Color32 alpha is 255); nil is refused. The resolver is /// part of the default chain, so a consumer /// registers nothing. /// @@ -49,7 +49,6 @@ public sealed class CultMathResolver : IFormatterResolver public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { - if (reader.IsNil) throw new MessagePackSerializationException($"{typeof(T).Name} cannot be read from nil."); return read(ref reader); } } @@ -171,7 +170,7 @@ public sealed class CultMathResolver : IFormatterResolver Shape((ref MessagePackWriter w, quaternion v) => Put(ref w, stackalloc float[] { v.x, v.y, v.z, v.w }), (ref MessagePackReader r) => { Span s = stackalloc float[4]; Singles(ref r, s); return new quaternion(s[0], s[1], s[2], s[3]); }), Shape((ref MessagePackWriter w, Color32 v) => Put(ref w, stackalloc byte[] { v.r, v.g, v.b, v.a }), - (ref MessagePackReader r) => { Span s = stackalloc byte[4]; Bytes(ref r, s); return new Color32(s[0], s[1], s[2], s[3]); }), + (ref MessagePackReader r) => { Span s = stackalloc byte[4]; s[3] = 255; Bytes(ref r, s); return new Color32(s[0], s[1], s[2], s[3]); }), Shape((ref MessagePackWriter w, Random v) => { w.WriteArrayHeader(1); w.Write(v.state); }, (ref MessagePackReader r) => { diff --git a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs index 76f95d34..fb6f1489 100644 --- a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs +++ b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs @@ -5,8 +5,11 @@ using FluentAssertions; using GameCult.Caching.MessagePack; using CultMath; using MessagePack; +using MessagePack.Formatters; using NUnit.Framework; +[assembly: CultCacheFormatterResolver(typeof(GameCult.Caching.Tests.CultMathSerializationTests.Float2Override))] + namespace GameCult.Caching.Tests { public sealed class CultMathSerializationTests @@ -106,6 +109,61 @@ namespace GameCult.Caching.Tests Hex(new float2x2(1f, 2f, 3f, 4f)).Should().Be("9292CA3F800000CA4000000092CA40400000CA40800000"); } + + [Test] + public void EveryShape_IsPinnedByteForByte() + { + Hex(new double2(1d, 2d)).Should().Be("92CB3FF0000000000000CB4000000000000000"); + Hex(new double3(1d, 2d, 3d)).Should().Be("93CB3FF0000000000000CB4000000000000000CB4008000000000000"); + Hex(new int3(1, -2, 3)).Should().Be("9301FE03"); + Hex(new int4(1, 2, 3, -1)).Should().Be("94010203FF"); + Hex(new bool3(true, false, true)).Should().Be("93C3C2C3"); + Hex(new bool4(false, true, true, false)).Should().Be("94C2C3C3C2"); + Hex(new quaternion(1f, 2f, 3f, 4f)).Should().Be("94CA3F800000CA40000000CA40400000CA40800000"); + Hex(new Color32(255, 128, 0, 7)).Should().Be("94CCFFCC800007"); + Hex(new CultMath.Random(12345u)).Should().Be("91CD3039"); + Hex(new float3x3(1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f)).Should().Be( + "93" + + "93CA3F800000CA40000000CA40400000" + + "93CA40800000CA40A00000CA40C00000" + + "93CA40E00000CA41000000CA41100000"); + Hex(new CultCellular(new float4(1f, 2f, 3f, 4f), new float4(5f, 6f, 7f, 8f), 0.25f)).Should().Be( + "93" + + "94CA3F800000CA40000000CA40400000CA40800000" + + "94CA40A00000CA40C00000CA40E00000CA41000000" + + "CA3E800000"); + } + + [Test] + public void Color32_MissingAlphaDecodesAsOpaque() + { + Decode("93CCFFCC8000").Should().Be(new Color32(255, 128, 0, 255)); + Decode("94CCFFCC800007").Should().Be(new Color32(255, 128, 0, 7)); + } + + [Test] + public void ConsumerResolvers_WinOverCultMathResolver() + { + var overridden = CultDocumentMessagePackSerialization.OptionsFor(typeof(Float2Override).Assembly); + + MessagePackSerializer.Serialize(new float2(1f, 2f), overridden).Should().Equal(0xC3); + Hex(new float2(1f, 2f)).Should().Be("92CA3F800000CA40000000"); + } + + public sealed class Float2Override : IFormatterResolver + { + public static readonly Float2Override Instance = new(); + + public IMessagePackFormatter? GetFormatter() => + typeof(T) == typeof(float2) ? (IMessagePackFormatter)(object)Formatter.Instance : null; + + private sealed class Formatter : IMessagePackFormatter + { + public static readonly Formatter Instance = new(); + public void Serialize(ref MessagePackWriter writer, float2 value, MessagePackSerializerOptions options) => writer.Write(true); + public float2 Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => throw new NotSupportedException(); + } + } [Test] public void EveryPublicCultMathValueType_HasAFormatter() { From 20c30224fd70ff36cc9c46ff060ae6dcec68d884 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:57:31 +0200 Subject: [PATCH 14/64] Add canonical System.Text.Json converters for every CultMath value type CultMathJson is the JSON twin of CultMathResolver: vectors, quaternion and Color32 are objects keyed by field name, matrices are arrays of row objects, rect is {min,max}. Consumers add them with AddCultMathConverters(); CultLib owns no JSON options. CultCircle and CultSphere serialize and deserialize through them, with [JsonConstructor] so the struct constructor is used. The Caching README documents each MessagePack and JSON shape in one table. Co-Authored-By: Claude Sonnet 5 --- .../CultMathJson.cs | 143 ++++++++++++++++++ .../GameCult.Caching.MessagePack.csproj | 1 + src/GameCult.Caching.MessagePack/README.md | 24 +++ .../CultGeometryPrimitives.cs | 2 + .../CultMathJsonTests.cs | 104 +++++++++++++ .../GeometryDocumentTests.cs | 16 ++ 6 files changed, 290 insertions(+) create mode 100644 src/GameCult.Caching.MessagePack/CultMathJson.cs create mode 100644 tests/GameCult.Caching.Tests/CultMathJsonTests.cs diff --git a/src/GameCult.Caching.MessagePack/CultMathJson.cs b/src/GameCult.Caching.MessagePack/CultMathJson.cs new file mode 100644 index 00000000..7070d389 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultMathJson.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using CultMath; +using Random = CultMath.Random; + +namespace GameCult.Caching.MessagePack; + +/// +/// The canonical System.Text.Json shape of every public CultMath value type, the JSON twin of +/// . Vectors, quaternions and colors are objects keyed by their +/// field names ({"x":1,"y":2}), matrices are arrays of row objects, rect is +/// {"min":..,"max":..}, and CultCellular is {"nearest":..,"edge":..,"id":..}. +/// Reading skips unknown properties and leaves missing ones at zero (a missing Color32.a +/// is 255); a value that is not an object, or a matrix that is not an array of exactly its row count, is refused. +/// Register with ; JSON options are the consumer's own. +/// +public static class CultMathJson +{ + /// The CultMath types that have a converter, for coverage checks. + public static IReadOnlyCollection ConvertedTypes => Converters.Keys; + + /// Adds the CultMath converters to and returns it. + public static JsonSerializerOptions AddCultMathConverters(this JsonSerializerOptions options) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + foreach (var converter in Converters.Values) options.Converters.Add(converter); + return options; + } + + private sealed class ObjectShape : JsonConverter + { + private readonly Action write; + private readonly Func read; + + public ObjectShape(Action write, Func read) + { + this.write = write; + this.read = read; + } + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using var document = JsonDocument.ParseValue(ref reader); + return read(document.RootElement); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) => write(writer, value); + } + + private static KeyValuePair Shape(Action write, Func read) => + new(typeof(T), new ObjectShape(write, read)); + + private static JsonElement Obj(JsonElement e) + { + if (e.ValueKind != JsonValueKind.Object) throw new JsonException("Expected a JSON object."); + return e; + } + + private static JsonElement Rows(JsonElement e, int count) + { + if (e.ValueKind != JsonValueKind.Array || e.GetArrayLength() != count) + throw new JsonException($"Expected a JSON array of {count} rows."); + return e; + } + + private static bool Has(JsonElement e, string name, out JsonElement value) => Obj(e).TryGetProperty(name, out value); + private static float F(JsonElement e, string name) => Has(e, name, out var v) ? v.GetSingle() : 0f; + private static double D(JsonElement e, string name) => Has(e, name, out var v) ? v.GetDouble() : 0d; + private static int I(JsonElement e, string name) => Has(e, name, out var v) ? v.GetInt32() : 0; + private static bool B(JsonElement e, string name) => Has(e, name, out var v) && v.GetBoolean(); + private static JsonElement Member(JsonElement e, string name) => + Has(e, name, out var v) ? v : Empty; + private static readonly JsonElement Empty = JsonDocument.Parse("{}").RootElement; + + private static void PutFloat2(Utf8JsonWriter w, float2 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); } + private static void PutFloat3(Utf8JsonWriter w, float3 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); } + private static void PutFloat4(Utf8JsonWriter w, float4 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); } + private static float2 GetFloat2(JsonElement e) => new(F(e, "x"), F(e, "y")); + private static float3 GetFloat3(JsonElement e) => new(F(e, "x"), F(e, "y"), F(e, "z")); + private static float4 GetFloat4(JsonElement e) => new(F(e, "x"), F(e, "y"), F(e, "z"), F(e, "w")); + + private static readonly Dictionary Converters = Build(); + + private static Dictionary Build() + { + var shapes = new[] + { + Shape(PutFloat2, GetFloat2), + Shape(PutFloat3, GetFloat3), + Shape(PutFloat4, GetFloat4), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); }, + e => new double2(D(e, "x"), D(e, "y"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); }, + e => new double3(D(e, "x"), D(e, "y"), D(e, "z"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); }, + e => new int2(I(e, "x"), I(e, "y"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); }, + e => new int3(I(e, "x"), I(e, "y"), I(e, "z"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); }, + e => new int4(I(e, "x"), I(e, "y"), I(e, "z"), I(e, "w"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteEndObject(); }, + e => new bool2(B(e, "x"), B(e, "y"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteBoolean("z", v.z); w.WriteEndObject(); }, + e => new bool3(B(e, "x"), B(e, "y"), B(e, "z"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteBoolean("z", v.z); w.WriteBoolean("w", v.w); w.WriteEndObject(); }, + e => new bool4(B(e, "x"), B(e, "y"), B(e, "z"), B(e, "w"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); }, + e => new quaternion(F(e, "x"), F(e, "y"), F(e, "z"), F(e, "w"))), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("r", v.r); w.WriteNumber("g", v.g); w.WriteNumber("b", v.b); w.WriteNumber("a", v.a); w.WriteEndObject(); }, + e => new Color32( + Has(e, "r", out var r) ? r.GetByte() : (byte)0, + Has(e, "g", out var g) ? g.GetByte() : (byte)0, + Has(e, "b", out var b) ? b.GetByte() : (byte)0, + Has(e, "a", out var a) ? a.GetByte() : (byte)255)), + Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("state", v.state); w.WriteEndObject(); }, + e => new Random { state = Has(e, "state", out var s) ? s.GetUInt32() : 0u }), + + // Compositions: matrices are arrays of row objects, rect is {min, max}, CultCellular is {nearest, edge, id}. + Shape((w, v) => { w.WriteStartArray(); PutFloat2(w, v[0]); PutFloat2(w, v[1]); w.WriteEndArray(); }, + e => + { + var rows = Rows(e, 2); + return new float2x2(GetFloat2(rows[0]), GetFloat2(rows[1])); + }), + Shape((w, v) => { w.WriteStartArray(); PutFloat3(w, v[0]); PutFloat3(w, v[1]); PutFloat3(w, v[2]); w.WriteEndArray(); }, + e => + { + var rows = Rows(e, 3); + return new float3x3(GetFloat3(rows[0]), GetFloat3(rows[1]), GetFloat3(rows[2])); + }), + Shape((w, v) => { w.WriteStartObject(); w.WritePropertyName("min"); PutFloat2(w, v.min); w.WritePropertyName("max"); PutFloat2(w, v.max); w.WriteEndObject(); }, + e => new rect(GetFloat2(Member(e, "min")), GetFloat2(Member(e, "max")))), + Shape((w, v) => { w.WriteStartObject(); w.WritePropertyName("nearest"); PutFloat4(w, v.nearest); w.WritePropertyName("edge"); PutFloat4(w, v.edge); w.WriteNumber("id", v.id); w.WriteEndObject(); }, + e => new CultCellular(GetFloat4(Member(e, "nearest")), GetFloat4(Member(e, "edge")), F(e, "id"))), + }; + + var map = new Dictionary(); + foreach (var shape in shapes) map.Add(shape.Key, shape.Value); + return map; + } +} diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index a37232a9..c2588e24 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -8,6 +8,7 @@ + diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index 277d0cb2..ae4d08a4 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -57,3 +57,27 @@ await cache.PullAllBackingStoresAsync(); persisted through hand-written MessagePack array layouts in this package. - The raw CultNet document lane should treat these payload bytes as already blessed, not decode and re-encode them for sport. + +## CultMath Value Shapes + +Every public CultMath value type has one MessagePack shape (`CultMathResolver`, +in the default chain) and one JSON shape (`CultMathJson`, added to a consumer's +options with `new JsonSerializerOptions().AddCultMathConverters()`; CultLib owns +no JSON options of its own). MessagePack decoding skips extra elements, leaves +missing ones at zero (`Color32` alpha at 255) and refuses nil. JSON reading +skips unknown properties and leaves missing ones at zero. A consumer resolver or +converter registered ahead of these wins. + +| Type | MessagePack | JSON | +| --- | --- | --- | +| `float2`, `double2`, `int2`, `bool2` | `[x, y]` | `{"x":..,"y":..}` | +| `float3`, `double3`, `int3`, `bool3` | `[x, y, z]` | `{"x":..,"y":..,"z":..}` | +| `float4`, `int4`, `bool4`, `quaternion` | `[x, y, z, w]` | `{"x":..,"y":..,"z":..,"w":..}` | +| `Color32` | `[r, g, b, a]` as uint8 | `{"r":..,"g":..,"b":..,"a":..}` | +| `Random` | `[state]` | `{"state":..}` | +| `float2x2`, `float3x3` | array of row vectors (row-major) | array of row objects | +| `rect` | `[min, max]` of `float2` | `{"min":{..},"max":{..}}` | +| `CultCellular` | `[nearest, edge, id]` | `{"nearest":{..},"edge":{..},"id":..}` | + +Floats are MessagePack float32 (`0xCA`), doubles float64 (`0xCB`), integers the +shortest MessagePack integer. diff --git a/src/GameCult.Geometry/CultGeometryPrimitives.cs b/src/GameCult.Geometry/CultGeometryPrimitives.cs index dafcca68..580f2b71 100644 --- a/src/GameCult.Geometry/CultGeometryPrimitives.cs +++ b/src/GameCult.Geometry/CultGeometryPrimitives.cs @@ -12,6 +12,7 @@ namespace GameCult.Geometry public readonly struct CultCircle : IEquatable { /// Creates a circle. + [JsonConstructor] public CultCircle(float2 center, float radius) { if (radius < 0f) @@ -75,6 +76,7 @@ namespace GameCult.Geometry public readonly struct CultSphere : IEquatable { /// Creates a sphere. + [JsonConstructor] public CultSphere(float3 center, float radius) { if (radius < 0f) diff --git a/tests/GameCult.Caching.Tests/CultMathJsonTests.cs b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs new file mode 100644 index 00000000..64702b74 --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs @@ -0,0 +1,104 @@ +using System; +using System.Linq; +using System.Text.Json; +using CultMath; +using FluentAssertions; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + public sealed class CultMathJsonTests + { + private static readonly JsonSerializerOptions Options = new JsonSerializerOptions().AddCultMathConverters(); + + private static string Json(T value) => JsonSerializer.Serialize(value, Options); + + private static T Parse(string json) => JsonSerializer.Deserialize(json, Options)!; + + [Test] + public void EveryShape_IsPinned() + { + Json(new float2(1f, 2f)).Should().Be("""{"x":1,"y":2}"""); + Json(new float3(1f, 2f, 3f)).Should().Be("""{"x":1,"y":2,"z":3}"""); + Json(new float4(1f, 2f, 3f, 4f)).Should().Be("""{"x":1,"y":2,"z":3,"w":4}"""); + Json(new double2(1d, 2.5d)).Should().Be("""{"x":1,"y":2.5}"""); + Json(new double3(1d, 2d, 3d)).Should().Be("""{"x":1,"y":2,"z":3}"""); + Json(new int2(3, -4)).Should().Be("""{"x":3,"y":-4}"""); + Json(new int3(1, 2, 3)).Should().Be("""{"x":1,"y":2,"z":3}"""); + Json(new int4(1, 2, 3, 4)).Should().Be("""{"x":1,"y":2,"z":3,"w":4}"""); + Json(new bool2(true, false)).Should().Be("""{"x":true,"y":false}"""); + Json(new bool3(true, false, true)).Should().Be("""{"x":true,"y":false,"z":true}"""); + Json(new bool4(true, false, false, true)).Should().Be("""{"x":true,"y":false,"z":false,"w":true}"""); + Json(new quaternion(0f, 0f, 0f, 1f)).Should().Be("""{"x":0,"y":0,"z":0,"w":1}"""); + Json(new Color32(255, 128, 0, 7)).Should().Be("""{"r":255,"g":128,"b":0,"a":7}"""); + Json(new CultMath.Random(12345u)).Should().Be("""{"state":12345}"""); + Json(new float2x2(1f, 2f, 3f, 4f)).Should().Be("""[{"x":1,"y":2},{"x":3,"y":4}]"""); + Json(new float3x3(1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f)).Should().Be( + """[{"x":1,"y":2,"z":3},{"x":4,"y":5,"z":6},{"x":7,"y":8,"z":9}]"""); + Json(new rect(5f, 7f, -1f, -2f)).Should().Be("""{"min":{"x":-1,"y":-2},"max":{"x":5,"y":7}}"""); + Json(new CultCellular(new float4(1f, 2f, 3f, 4f), new float4(5f, 6f, 7f, 8f), 0.25f)).Should().Be( + """{"nearest":{"x":1,"y":2,"z":3,"w":4},"edge":{"x":5,"y":6,"z":7,"w":8},"id":0.25}"""); + } + + [Test] + public void EveryShape_RoundTrips() + { + RoundTrip(new float2(1.5f, -2f)); + RoundTrip(new float3(1f, 2f, 3f)); + RoundTrip(new float4(1f, 2f, 3f, 4f)); + RoundTrip(new double2(1e300, -2.5)); + RoundTrip(new double3(1, 2, 3)); + RoundTrip(new int2(-7, 900000)); + RoundTrip(new int3(1, -2, 3)); + RoundTrip(new int4(1, 2, 3, int.MinValue)); + RoundTrip(new bool2(true, false)); + RoundTrip(new bool3(false, true, true)); + RoundTrip(new bool4(true, false, false, true)); + RoundTrip(new quaternion(0.1f, 0.2f, 0.3f, 0.9f)); + RoundTrip(new Color32(255, 128, 0, 7)); + RoundTrip(new CultMath.Random(12345u)); + RoundTrip(new float2x2(1f, 2f, 3f, 4f)); + RoundTrip(new float3x3(1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f)); + RoundTrip(new rect(-1f, -2f, 5f, 7f)); + RoundTrip(new CultCellular(new float4(1, 2, 3, 4), new float4(5, 6, 7, 8), 0.25f)); + } + + [Test] + public void Reading_SkipsUnknownPropertiesAndZeroesMissingOnes() + { + Parse("""{"z":3,"extra":[1,2],"x":1}""").Should().Be(new float3(1f, 0f, 3f)); + Parse("""{"r":1,"g":2,"b":3}""").Should().Be(new Color32(1, 2, 3, 255)); + Parse("null").Should().BeNull(); + } + + [Test] + public void Reading_RefusesShapesThatAreNotTheType() + { + Action array = () => Parse("[1,2]"); + array.Should().Throw(); + Action shortMatrix = () => Parse("""[{"x":1,"y":2}]"""); + shortMatrix.Should().Throw(); + Action nil = () => Parse("null"); + nil.Should().Throw(); + } + + [Test] + public void EveryPublicCultMathValueType_HasAConverter() + { + var valueTypes = typeof(float2).Assembly.GetExportedTypes() + .Where(t => t.IsValueType && !t.IsEnum) + .ToHashSet(); + + valueTypes.Except(CultMathJson.ConvertedTypes).Should().BeEmpty(); + CultMathJson.ConvertedTypes.Except(valueTypes).Should().BeEmpty(); + foreach (var type in valueTypes) + { + var json = JsonSerializer.Serialize(Activator.CreateInstance(type), type, Options); + JsonSerializer.Deserialize(json, type, Options).Should().Be(Activator.CreateInstance(type)); + } + } + + private static void RoundTrip(T value) => Parse(Json(value)).Should().Be(value); + } +} diff --git a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs index daae992a..8741f891 100644 --- a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs +++ b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs @@ -47,6 +47,22 @@ namespace GameCult.Geometry.Tests sphere.XyCircle.Bounds.Should().Be(new rect(-4f, -3f, 6f, 7f)); } + [Test] + public void GeometryPrimitives_JsonOmitsComputedConvenienceProperties() + { + var options = new JsonSerializerOptions().AddCultMathConverters(); + var sphere = new CultSphere(new float3(1f, 2f, 3f), 5f); + var circle = new CultCircle(new float2(1f, 2f), 3f); + + var sphereJson = JsonSerializer.Serialize(sphere, options); + var circleJson = JsonSerializer.Serialize(circle, options); + + sphereJson.Should().Be("""{"Center":{"x":1,"y":2,"z":3},"Radius":5}"""); + circleJson.Should().Be("""{"Center":{"x":1,"y":2},"Radius":3}"""); + JsonSerializer.Deserialize(sphereJson, options).Should().Be(sphere); + JsonSerializer.Deserialize(circleJson, options).Should().Be(circle); + } + // The bytes CultCircle and CultSphere produced when they held CultVec2/CultVec3 (probed at 65311c9). [Test] public void GeometryPrimitives_KeepTheirPreCollapseBytes() From 9dcce987fd0eba713d07084826b32f110defea28 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:57:31 +0200 Subject: [PATCH 15/64] Update Mesh docs that still named CultVec2 and CultRect Co-Authored-By: Claude Sonnet 5 --- .../docs/cross-runtime-primitives-roadmap.md | 8 ++++---- src/GameCult.Mesh/docs/public-api.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md b/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md index 93fb3f3c..5b0048de 100644 --- a/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md +++ b/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md @@ -116,12 +116,12 @@ var aetheria = verse.Use(AetheriaRuntime.Schema); var viewport = await aetheria.Zone(zoneId) .Objects() .VisibleTo(controlledUnits) - .Within(CultRect.FromBounds(-400, -250, 400, 250)) + .Within(new rect(-400f, -250f, 400f, 250f)) .QueryAsync(); await aetheria.Entity(ravenId) .Pilot() - .MoveAsync(CultVec2.Right, claim: "pilot-control"); + .MoveAsync(new float2(1f, 0f), claim: "pilot-control"); ``` Current seed: @@ -134,7 +134,7 @@ var verse = await CultMesh.ConnectVerseAsync( new[] { new CultMeshAuthorityClaim("pilot-control", shardId: "zone:raven") }); var aetheria = verse.Use(context => new AetheriaGeneratedFacade(context)); -await aetheria.Entity(ravenId).Pilot.MoveAsync(CultVec2.Right); +await aetheria.Entity(ravenId).Pilot.MoveAsync(new float2(1f, 0f)); ``` ```ts @@ -177,7 +177,7 @@ let viewport = aetheria .zone(zone_id) .objects() .visible_to(&controlled_units) - .within(CultRect::from_bounds(-400.0, -250.0, 400.0, 250.0)) + .within(rect::new(-400.0, -250.0, 400.0, 250.0)) .query() .await?; ``` diff --git a/src/GameCult.Mesh/docs/public-api.md b/src/GameCult.Mesh/docs/public-api.md index df1735b6..e9f19ab7 100644 --- a/src/GameCult.Mesh/docs/public-api.md +++ b/src/GameCult.Mesh/docs/public-api.md @@ -407,8 +407,8 @@ var view = new CultMeshNativeSliceViewDescriptor( rowCount, new[] { - CultMeshNativeSliceColumn.For("position"), - CultMeshNativeSliceColumn.For("velocity") + CultMeshNativeSliceColumn.For("position"), + CultMeshNativeSliceColumn.For("velocity") }, new CultMeshRouteHint(CultMeshLocalityKind.SharedMemory)); ``` From 9b23b33f8c35afc76204659807925ff4e924b94a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:11:47 +0200 Subject: [PATCH 16/64] Let Stryker mutate the variant name rule The is-not pattern binding made every mutant in RefuseVariantNameSharing a compile error, so Stryker removed the whole method from its run. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2a9de06e..d2115535 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2761,7 +2761,8 @@ namespace GameCult.Caching foreach (var variant in landing.Values.Where(stored => stored.Variant != null)) { - if (plan.Projections[variant.Key.Value].Name is not { } variantName) + var variantName = plan.Projections[variant.Key.Value].Name; + if (variantName == null) continue; for (var ancestor = Ancestor(variant.Variant!.BaseKey); ancestor != null; ancestor = ancestor.Variant == null ? null : Ancestor(ancestor.Variant.BaseKey)) From 9b5597e89c291ec42ac1f79ef6b0a01b6976c3bd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:20:17 +0200 Subject: [PATCH 17/64] Gate the CultLib Unity release on the CultMath package it depends on The package build now fails when the declared org.gamecult.cultmath version is ahead of the CultMath package, or when that package's CultMath.dll lacks a type CultMathResolver formats. It also drops the shipped-plugin guard that could not fire, since the output root is wiped first. Co-Authored-By: Claude Sonnet 5 --- scripts/build-unity-package.ps1 | 37 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/scripts/build-unity-package.ps1 b/scripts/build-unity-package.ps1 index 3258155f..49bf391f 100644 --- a/scripts/build-unity-package.ps1 +++ b/scripts/build-unity-package.ps1 @@ -53,6 +53,34 @@ if ($LASTEXITCODE -ne 0) { throw "CultLib Unity package failed the semver policy check (see docs/semver-policy.md)." } + +# Release gate: cultlib's Unity package resolves CultMath through the org.gamecult.cultmath +# dependency, so a cultlib release must not run ahead of the CultMath package it names. +# (1) The declared dependency version must not exceed the CultMath package's own version. +# (2) The CultMath.dll that package tracks must define every type CultMathResolver formats. +# Type names come from the resolver source (its Shape entries) and are looked up as +# NUL-delimited names in the assembly's metadata string heap, which is a name check, not a +# signature check; Windows PowerShell cannot load a netstandard2.1 assembly to reflect on it. +$cultMathPackageRoot = Join-Path $repoRoot "packages\cultmath\unity\org.gamecult.cultmath" +$cultMathDeclared = (Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json).dependencies.'org.gamecult.cultmath' +$cultMathAvailable = (Get-Content -LiteralPath (Join-Path $cultMathPackageRoot "package.json") -Raw | ConvertFrom-Json).version +if ([string]::IsNullOrWhiteSpace($cultMathDeclared)) { + throw "package.json must declare org.gamecult.cultmath, which supplies CultMath.dll." +} +if ([version]$cultMathDeclared -gt [version]$cultMathAvailable) { + throw "Release order: org.gamecult.cultlib depends on org.gamecult.cultmath $cultMathDeclared, but the CultMath package is at $cultMathAvailable. Release CultMath first." +} +$resolverSource = Get-Content -LiteralPath (Join-Path $repoRoot "src\GameCult.Caching.MessagePack\CultMathResolver.cs") -Raw +$resolverTypes = @([regex]::Matches($resolverSource, '\bShape<(?!T>)(\w+)>\(') | ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique) +if ($resolverTypes.Count -eq 0) { + throw "Found no CultMath types in CultMathResolver.cs; the release gate cannot check the CultMath package." +} +$cultMathBytes = [IO.File]::ReadAllBytes((Join-Path $cultMathPackageRoot "Runtime\Plugins\CultMath.dll")) +$cultMathText = [Text.Encoding]::GetEncoding(28591).GetString($cultMathBytes) +$missingTypes = @($resolverTypes | Where-Object { -not $cultMathText.Contains("`0$_`0") }) +if ($missingTypes.Count -gt 0) { + throw "The CultMath.dll in org.gamecult.cultmath $cultMathAvailable lacks types CultMathResolver formats: $($missingTypes -join ', '). Release CultMath first." +} # The tracked DLLs and pdbs are committed beside their source, so none may name a commit or a worktree: # Source Link writes the commit SHA into the pdb, the informational version carries it too, and each DLL # carries its pdb's content id. ContinuousIntegrationBuild maps the local source path to /_/. @@ -137,10 +165,6 @@ $declaredAssemblies = @($expectedAssemblies + $externalAssemblies | Sort-Object) if (($referencedAssemblies -join "|") -ne ($declaredAssemblies -join "|")) { throw "GameCult.CultLib.asmdef precompiledReferences differ from the shipped plus external assembly lists." } -$cultMathDependency = (Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json).dependencies.'org.gamecult.cultmath' -if ([string]::IsNullOrWhiteSpace($cultMathDependency)) { - throw "package.json must declare org.gamecult.cultmath, which supplies CultMath.dll." -} $publishedByName = @{} foreach ($assembly in Get-ChildItem -LiteralPath $publishRoot -Filter "*.dll") { $publishedByName[$assembly.Name] = $assembly @@ -167,11 +191,6 @@ foreach ($assemblyName in $expectedAssemblies) { Copy-Item -LiteralPath $pdb -Destination $pluginRoot } } -foreach ($external in $externalAssemblies) { - if (Test-Path -LiteralPath (Join-Path $pluginRoot $external)) { - throw "CultLib Unity package must not ship $external; org.gamecult.cultmath owns it." - } -} $nativePluginRoot = Join-Path $pluginRoot "x86_64" New-Item -ItemType Directory -Force -Path $nativePluginRoot | Out-Null Copy-Item -LiteralPath (Join-Path $quicNativeRoot "gamecult_mesh_quic_native.dll") -Destination $nativePluginRoot From 817c2627b8903fa44ec3d0996d4c54051402ed2f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:29:02 +0200 Subject: [PATCH 18/64] Rebuild the CultMath JSON converters on a streaming reader with STJ number semantics One Scalar plus Node description per type now drives reading, writing and the dictionary-key spelling, replacing the per-value JsonDocument parse. NaN and the infinities follow JsonNumberHandling (named literals or JsonException), floats beyond float.MaxValue are refused instead of becoming infinity, out-of-range components throw JsonException instead of FormatException, property matching honours PropertyNameCaseInsensitive, and every type works as a dictionary key spelled as comma-joined components. Co-Authored-By: Claude Sonnet 5 --- .../CultMathJson.cs | 513 +++++++++++++++--- src/GameCult.Caching.MessagePack/README.md | 11 +- .../CultMathJsonTests.cs | 126 +++++ 3 files changed, 567 insertions(+), 83 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultMathJson.cs b/src/GameCult.Caching.MessagePack/CultMathJson.cs index 7070d389..def7ab9a 100644 --- a/src/GameCult.Caching.MessagePack/CultMathJson.cs +++ b/src/GameCult.Caching.MessagePack/CultMathJson.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; using CultMath; @@ -12,8 +13,26 @@ namespace GameCult.Caching.MessagePack; /// . Vectors, quaternions and colors are objects keyed by their /// field names ({"x":1,"y":2}), matrices are arrays of row objects, rect is /// {"min":..,"max":..}, and CultCellular is {"nearest":..,"edge":..,"id":..}. +/// Property names are fixed: PropertyNamingPolicy does not rename them, and +/// PropertyNameCaseInsensitive makes reading case-insensitive. +/// /// Reading skips unknown properties and leaves missing ones at zero (a missing Color32.a -/// is 255); a value that is not an object, or a matrix that is not an array of exactly its row count, is refused. +/// is 255). A wrong token, a number outside its component type (including a float beyond +/// , which is refused rather than becoming infinity), or a matrix +/// that is not an array of exactly its row count throws . +/// +/// +/// Numbers follow as System.Text.Json's own +/// float does: NaN and the infinities need AllowNamedFloatingPointLiterals (written and +/// read as "NaN", "Infinity", "-Infinity") and are otherwise refused with +/// ; AllowReadingFromString reads quoted numbers; +/// WriteAsString writes quoted numbers. +/// +/// +/// Every type is also a dictionary key: the flattened components joined by commas in +/// declaration order, invariant culture, shortest round-trip floats ("1,2", +/// "true,false", matrices row by row, rect as "minX,minY,maxX,maxY"). +/// /// Register with ; JSON options are the consumer's own. /// public static class CultMathJson @@ -29,111 +48,443 @@ public static class CultMathJson return options; } - private sealed class ObjectShape : JsonConverter - { - private readonly Action write; - private readonly Func read; + private static bool Named(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.AllowNamedFloatingPointLiterals) != 0; + private static bool FromString(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.AllowReadingFromString) != 0; + private static bool AsString(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.WriteAsString) != 0; - public ObjectShape(Action write, Func read) + private static string Invalid(string what, string got) => $"{what} cannot be read from '{got}'."; + + // One component kind. Every CultMath type is a flat list of one kind, so a Scalar plus a Node tree describes it. + private abstract class Scalar + { + public virtual S[] Defaults(int count) => new S[count]; + public abstract void Write(Utf8JsonWriter w, S value, JsonSerializerOptions o); + public abstract S Read(ref Utf8JsonReader r, JsonSerializerOptions o); + public abstract string Text(S value, JsonSerializerOptions o); + public abstract S Parse(string text, JsonSerializerOptions o); + } + + private sealed class FloatScalar : Scalar + { + public static readonly FloatScalar Instance = new(); + + private static bool IsLiteral(string s) => s == "NaN" || s == "Infinity" || s == "-Infinity"; + + private static string Literal(float v) => float.IsNaN(v) ? "NaN" : v > 0f ? "Infinity" : "-Infinity"; + + public override void Write(Utf8JsonWriter w, float v, JsonSerializerOptions o) { - this.write = write; - this.read = read; + if (!float.IsFinite(v)) + { + if (!Named(o)) throw new JsonException("A non-finite float needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); + w.WriteStringValue(Literal(v)); + } + else if (AsString(o)) w.WriteStringValue(v.ToString("R", CultureInfo.InvariantCulture)); + else w.WriteNumberValue(v); + } + + public override float Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!r.TryGetSingle(out var v) || float.IsInfinity(v)) throw new JsonException("The number is outside the float range."); + return v; + } + + if (r.TokenType == JsonTokenType.String) + { + var text = r.GetString()!; + if (!FromString(o) && !(Named(o) && IsLiteral(text))) throw new JsonException($"Expected a number, found the string '{text}'."); + return Parse(text, o); + } + + throw new JsonException($"Expected a number, found {r.TokenType}."); + } + + public override string Text(float v, JsonSerializerOptions o) + { + if (!float.IsFinite(v) && !Named(o)) throw new JsonException("A non-finite float needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); + return v.ToString("R", CultureInfo.InvariantCulture); + } + + public override float Parse(string text, JsonSerializerOptions o) + { + if (Named(o)) + { + if (text == "NaN") return float.NaN; + if (text == "Infinity") return float.PositiveInfinity; + if (text == "-Infinity") return float.NegativeInfinity; + } + + if (float.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var v) && float.IsFinite(v)) return v; + throw new JsonException(Invalid("A float", text)); + } + } + + private sealed class DoubleScalar : Scalar + { + public static readonly DoubleScalar Instance = new(); + + private static bool IsLiteral(string s) => s == "NaN" || s == "Infinity" || s == "-Infinity"; + + private static string Literal(double v) => double.IsNaN(v) ? "NaN" : v > 0d ? "Infinity" : "-Infinity"; + + public override void Write(Utf8JsonWriter w, double v, JsonSerializerOptions o) + { + if (!double.IsFinite(v)) + { + if (!Named(o)) throw new JsonException("A non-finite double needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); + w.WriteStringValue(Literal(v)); + } + else if (AsString(o)) w.WriteStringValue(v.ToString("R", CultureInfo.InvariantCulture)); + else w.WriteNumberValue(v); + } + + public override double Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!r.TryGetDouble(out var v) || double.IsInfinity(v)) throw new JsonException("The number is outside the double range."); + return v; + } + + if (r.TokenType == JsonTokenType.String) + { + var text = r.GetString()!; + if (!FromString(o) && !(Named(o) && IsLiteral(text))) throw new JsonException($"Expected a number, found the string '{text}'."); + return Parse(text, o); + } + + throw new JsonException($"Expected a number, found {r.TokenType}."); + } + + public override string Text(double v, JsonSerializerOptions o) + { + if (!double.IsFinite(v) && !Named(o)) throw new JsonException("A non-finite double needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); + return v.ToString("R", CultureInfo.InvariantCulture); + } + + public override double Parse(string text, JsonSerializerOptions o) + { + if (Named(o)) + { + if (text == "NaN") return double.NaN; + if (text == "Infinity") return double.PositiveInfinity; + if (text == "-Infinity") return double.NegativeInfinity; + } + + if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var v) && double.IsFinite(v)) return v; + throw new JsonException(Invalid("A double", text)); + } + } + + private sealed class IntScalar : Scalar + { + public static readonly IntScalar Instance = new(); + + public override void Write(Utf8JsonWriter w, int v, JsonSerializerOptions o) + { + if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); + } + + public override int Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!r.TryGetInt32(out var v)) throw new JsonException("The number is not an int."); + return v; + } + + if (r.TokenType == JsonTokenType.String && FromString(o)) return Parse(r.GetString()!, o); + throw new JsonException($"Expected an integer, found {r.TokenType}."); + } + + public override string Text(int v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); + + public override int Parse(string text, JsonSerializerOptions o) => + int.TryParse(text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("An int", text)); + } + + private sealed class UIntScalar : Scalar + { + public static readonly UIntScalar Instance = new(); + + public override void Write(Utf8JsonWriter w, uint v, JsonSerializerOptions o) + { + if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); + } + + public override uint Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!r.TryGetUInt32(out var v)) throw new JsonException("The number is not a uint."); + return v; + } + + if (r.TokenType == JsonTokenType.String && FromString(o)) return Parse(r.GetString()!, o); + throw new JsonException($"Expected an unsigned integer, found {r.TokenType}."); + } + + public override string Text(uint v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); + + public override uint Parse(string text, JsonSerializerOptions o) => + uint.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("A uint", text)); + } + + private sealed class ByteScalar : Scalar + { + public static readonly ByteScalar Instance = new(); + + public override byte[] Defaults(int count) + { + var defaults = new byte[count]; + if (count == 4) defaults[3] = 255; // Color32 alpha + return defaults; + } + + public override void Write(Utf8JsonWriter w, byte v, JsonSerializerOptions o) + { + if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); + } + + public override byte Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!r.TryGetByte(out var v)) throw new JsonException("The number is not a byte."); + return v; + } + + if (r.TokenType == JsonTokenType.String && FromString(o)) return Parse(r.GetString()!, o); + throw new JsonException($"Expected a byte, found {r.TokenType}."); + } + + public override string Text(byte v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); + + public override byte Parse(string text, JsonSerializerOptions o) => + byte.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("A byte", text)); + } + + private sealed class BoolScalar : Scalar + { + public static readonly BoolScalar Instance = new(); + + public override void Write(Utf8JsonWriter w, bool v, JsonSerializerOptions o) => w.WriteBooleanValue(v); + + public override bool Read(ref Utf8JsonReader r, JsonSerializerOptions o) => + r.TokenType == JsonTokenType.True ? true + : r.TokenType == JsonTokenType.False ? false + : throw new JsonException($"Expected true or false, found {r.TokenType}."); + + public override string Text(bool v, JsonSerializerOptions o) => v ? "true" : "false"; + + public override bool Parse(string text, JsonSerializerOptions o) => + text == "true" ? true : text == "false" ? false : throw new JsonException(Invalid("A bool", text)); + } + + // The JSON structure over a flat component list: leaves consume one component each, in order. + private abstract class Node + { + public int Count { get; protected set; } + public abstract void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o); + public abstract void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o); + } + + private sealed class Leaf : Node + { + public Leaf() { Count = 1; } + + public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) => scalar.Write(w, flat[offset], o); + + public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) => flat[offset] = scalar.Read(ref r, o); + } + + private sealed class Fields : Node + { + private readonly string[] names; + private readonly Node[] children; + private readonly int[] offsets; + + public Fields(params (string Name, Node Child)[] fields) + { + names = new string[fields.Length]; + children = new Node[fields.Length]; + offsets = new int[fields.Length]; + for (var i = 0; i < fields.Length; i++) + { + names[i] = fields[i].Name; + children[i] = fields[i].Child; + offsets[i] = Count; + Count += fields[i].Child.Count; + } + } + + public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + { + w.WriteStartObject(); + for (var i = 0; i < names.Length; i++) + { + w.WritePropertyName(names[i]); + children[i].Write(w, flat, offset + offsets[i], scalar, o); + } + + w.WriteEndObject(); + } + + public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + { + if (r.TokenType != JsonTokenType.StartObject) throw new JsonException($"Expected a JSON object, found {r.TokenType}."); + var comparison = o.PropertyNameCaseInsensitive ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + while (r.Read() && r.TokenType == JsonTokenType.PropertyName) + { + var name = r.GetString()!; + var index = Array.FindIndex(names, n => string.Equals(n, name, comparison)); + r.Read(); + if (index < 0) r.Skip(); else children[index].Read(ref r, flat, offset + offsets[index], scalar, o); + } + } + } + + private sealed class Rows : Node + { + private readonly Node row; + private readonly int rows; + + public Rows(int rows, Node row) + { + this.rows = rows; + this.row = row; + Count = rows * row.Count; + } + + public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + { + w.WriteStartArray(); + for (var i = 0; i < rows; i++) row.Write(w, flat, offset + (i * row.Count), scalar, o); + w.WriteEndArray(); + } + + public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + { + if (r.TokenType != JsonTokenType.StartArray) throw new JsonException($"Expected a JSON array of {rows} rows, found {r.TokenType}."); + for (var i = 0; i < rows; i++) + { + if (!r.Read() || r.TokenType == JsonTokenType.EndArray) throw new JsonException($"Expected {rows} rows."); + row.Read(ref r, flat, offset + (i * row.Count), scalar, o); + } + + if (!r.Read() || r.TokenType != JsonTokenType.EndArray) throw new JsonException($"Expected {rows} rows."); + } + } + + private sealed class Shape : JsonConverter + { + private readonly Scalar scalar; + private readonly Node root; + private readonly Func make; + private readonly Action flatten; + + public Shape(Scalar scalar, Node root, Func make, Action flatten) + { + this.scalar = scalar; + this.root = root; + this.make = make; + this.flatten = flatten; } public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - using var document = JsonDocument.ParseValue(ref reader); - return read(document.RootElement); + var flat = scalar.Defaults(root.Count); + root.Read(ref reader, flat, 0, scalar, options); + return make(flat); } - public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) => write(writer, value); + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + var flat = new S[root.Count]; + flatten(value, flat); + root.Write(writer, flat, 0, scalar, options); + } + + public override T ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var parts = (reader.GetString() ?? throw new JsonException("A dictionary key cannot be null.")).Split(','); + if (parts.Length != root.Count) throw new JsonException($"A {typeof(T).Name} key has {root.Count} comma-separated components."); + var flat = new S[root.Count]; + for (var i = 0; i < flat.Length; i++) flat[i] = scalar.Parse(parts[i], options); + return make(flat); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + var flat = new S[root.Count]; + flatten(value, flat); + writer.WritePropertyName(string.Join(",", Array.ConvertAll(flat, c => scalar.Text(c, options)))); + } } - private static KeyValuePair Shape(Action write, Func read) => - new(typeof(T), new ObjectShape(write, read)); + private static Node Val() => new Leaf(); - private static JsonElement Obj(JsonElement e) - { - if (e.ValueKind != JsonValueKind.Object) throw new JsonException("Expected a JSON object."); - return e; - } + private static Node Xy() => new Fields(("x", Val()), ("y", Val())); + private static Node Xyz() => new Fields(("x", Val()), ("y", Val()), ("z", Val())); + private static Node Xyzw() => new Fields(("x", Val()), ("y", Val()), ("z", Val()), ("w", Val())); - private static JsonElement Rows(JsonElement e, int count) - { - if (e.ValueKind != JsonValueKind.Array || e.GetArrayLength() != count) - throw new JsonException($"Expected a JSON array of {count} rows."); - return e; - } - - private static bool Has(JsonElement e, string name, out JsonElement value) => Obj(e).TryGetProperty(name, out value); - private static float F(JsonElement e, string name) => Has(e, name, out var v) ? v.GetSingle() : 0f; - private static double D(JsonElement e, string name) => Has(e, name, out var v) ? v.GetDouble() : 0d; - private static int I(JsonElement e, string name) => Has(e, name, out var v) ? v.GetInt32() : 0; - private static bool B(JsonElement e, string name) => Has(e, name, out var v) && v.GetBoolean(); - private static JsonElement Member(JsonElement e, string name) => - Has(e, name, out var v) ? v : Empty; - private static readonly JsonElement Empty = JsonDocument.Parse("{}").RootElement; - - private static void PutFloat2(Utf8JsonWriter w, float2 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); } - private static void PutFloat3(Utf8JsonWriter w, float3 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); } - private static void PutFloat4(Utf8JsonWriter w, float4 v) { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); } - private static float2 GetFloat2(JsonElement e) => new(F(e, "x"), F(e, "y")); - private static float3 GetFloat3(JsonElement e) => new(F(e, "x"), F(e, "y"), F(e, "z")); - private static float4 GetFloat4(JsonElement e) => new(F(e, "x"), F(e, "y"), F(e, "z"), F(e, "w")); + private static KeyValuePair Add( + Scalar scalar, Node root, Func make, Action flatten) => + new(typeof(T), new Shape(scalar, root, make, flatten)); private static readonly Dictionary Converters = Build(); private static Dictionary Build() { + var f = FloatScalar.Instance; var shapes = new[] { - Shape(PutFloat2, GetFloat2), - Shape(PutFloat3, GetFloat3), - Shape(PutFloat4, GetFloat4), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); }, - e => new double2(D(e, "x"), D(e, "y"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); }, - e => new double3(D(e, "x"), D(e, "y"), D(e, "z"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteEndObject(); }, - e => new int2(I(e, "x"), I(e, "y"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteEndObject(); }, - e => new int3(I(e, "x"), I(e, "y"), I(e, "z"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); }, - e => new int4(I(e, "x"), I(e, "y"), I(e, "z"), I(e, "w"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteEndObject(); }, - e => new bool2(B(e, "x"), B(e, "y"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteBoolean("z", v.z); w.WriteEndObject(); }, - e => new bool3(B(e, "x"), B(e, "y"), B(e, "z"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteBoolean("x", v.x); w.WriteBoolean("y", v.y); w.WriteBoolean("z", v.z); w.WriteBoolean("w", v.w); w.WriteEndObject(); }, - e => new bool4(B(e, "x"), B(e, "y"), B(e, "z"), B(e, "w"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("x", v.x); w.WriteNumber("y", v.y); w.WriteNumber("z", v.z); w.WriteNumber("w", v.w); w.WriteEndObject(); }, - e => new quaternion(F(e, "x"), F(e, "y"), F(e, "z"), F(e, "w"))), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("r", v.r); w.WriteNumber("g", v.g); w.WriteNumber("b", v.b); w.WriteNumber("a", v.a); w.WriteEndObject(); }, - e => new Color32( - Has(e, "r", out var r) ? r.GetByte() : (byte)0, - Has(e, "g", out var g) ? g.GetByte() : (byte)0, - Has(e, "b", out var b) ? b.GetByte() : (byte)0, - Has(e, "a", out var a) ? a.GetByte() : (byte)255)), - Shape((w, v) => { w.WriteStartObject(); w.WriteNumber("state", v.state); w.WriteEndObject(); }, - e => new Random { state = Has(e, "state", out var s) ? s.GetUInt32() : 0u }), + Add(f, Xy(), a => new float2(a[0], a[1]), (v, a) => { a[0] = v.x; a[1] = v.y; }), + Add(f, Xyz(), a => new float3(a[0], a[1], a[2]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; }), + Add(f, Xyzw(), a => new float4(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; a[3] = v.w; }), + Add(f, Xyzw(), a => new quaternion(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; a[3] = v.w; }), + Add(DoubleScalar.Instance, Xy(), a => new double2(a[0], a[1]), (v, a) => { a[0] = v.x; a[1] = v.y; }), + Add(DoubleScalar.Instance, Xyz(), a => new double3(a[0], a[1], a[2]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; }), + Add(IntScalar.Instance, Xy(), a => new int2(a[0], a[1]), (v, a) => { a[0] = v.x; a[1] = v.y; }), + Add(IntScalar.Instance, Xyz(), a => new int3(a[0], a[1], a[2]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; }), + Add(IntScalar.Instance, Xyzw(), a => new int4(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; a[3] = v.w; }), + Add(BoolScalar.Instance, Xy(), a => new bool2(a[0], a[1]), (v, a) => { a[0] = v.x; a[1] = v.y; }), + Add(BoolScalar.Instance, Xyz(), a => new bool3(a[0], a[1], a[2]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; }), + Add(BoolScalar.Instance, Xyzw(), a => new bool4(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; a[3] = v.w; }), + Add(ByteScalar.Instance, + new Fields(("r", Val()), ("g", Val()), ("b", Val()), ("a", Val())), + a => new Color32(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.r; a[1] = v.g; a[2] = v.b; a[3] = v.a; }), + Add(UIntScalar.Instance, new Fields(("state", Val())), + a => new Random { state = a[0] }, (v, a) => a[0] = v.state), // Compositions: matrices are arrays of row objects, rect is {min, max}, CultCellular is {nearest, edge, id}. - Shape((w, v) => { w.WriteStartArray(); PutFloat2(w, v[0]); PutFloat2(w, v[1]); w.WriteEndArray(); }, - e => + Add(f, new Rows(2, Xy()), + a => new float2x2(a[0], a[1], a[2], a[3]), + (v, a) => { a[0] = v[0].x; a[1] = v[0].y; a[2] = v[1].x; a[3] = v[1].y; }), + Add(f, new Rows(3, Xyz()), + a => new float3x3(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8]), + (v, a) => { - var rows = Rows(e, 2); - return new float2x2(GetFloat2(rows[0]), GetFloat2(rows[1])); + for (var row = 0; row < 3; row++) + { + a[row * 3] = v[row].x; + a[(row * 3) + 1] = v[row].y; + a[(row * 3) + 2] = v[row].z; + } }), - Shape((w, v) => { w.WriteStartArray(); PutFloat3(w, v[0]); PutFloat3(w, v[1]); PutFloat3(w, v[2]); w.WriteEndArray(); }, - e => + Add(f, new Fields(("min", Xy()), ("max", Xy())), + a => new rect(a[0], a[1], a[2], a[3]), + (v, a) => { a[0] = v.min.x; a[1] = v.min.y; a[2] = v.max.x; a[3] = v.max.y; }), + Add(f, new Fields(("nearest", Xyzw()), ("edge", Xyzw()), ("id", Val())), + a => new CultCellular(new float4(a[0], a[1], a[2], a[3]), new float4(a[4], a[5], a[6], a[7]), a[8]), + (v, a) => { - var rows = Rows(e, 3); - return new float3x3(GetFloat3(rows[0]), GetFloat3(rows[1]), GetFloat3(rows[2])); + a[0] = v.nearest.x; a[1] = v.nearest.y; a[2] = v.nearest.z; a[3] = v.nearest.w; + a[4] = v.edge.x; a[5] = v.edge.y; a[6] = v.edge.z; a[7] = v.edge.w; + a[8] = v.id; }), - Shape((w, v) => { w.WriteStartObject(); w.WritePropertyName("min"); PutFloat2(w, v.min); w.WritePropertyName("max"); PutFloat2(w, v.max); w.WriteEndObject(); }, - e => new rect(GetFloat2(Member(e, "min")), GetFloat2(Member(e, "max")))), - Shape((w, v) => { w.WriteStartObject(); w.WritePropertyName("nearest"); PutFloat4(w, v.nearest); w.WritePropertyName("edge"); PutFloat4(w, v.edge); w.WriteNumber("id", v.id); w.WriteEndObject(); }, - e => new CultCellular(GetFloat4(Member(e, "nearest")), GetFloat4(Member(e, "edge")), F(e, "id"))), }; var map = new Dictionary(); diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index ae4d08a4..27e3ea57 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -65,8 +65,15 @@ in the default chain) and one JSON shape (`CultMathJson`, added to a consumer's options with `new JsonSerializerOptions().AddCultMathConverters()`; CultLib owns no JSON options of its own). MessagePack decoding skips extra elements, leaves missing ones at zero (`Color32` alpha at 255) and refuses nil. JSON reading -skips unknown properties and leaves missing ones at zero. A consumer resolver or -converter registered ahead of these wins. +skips unknown properties, leaves missing ones at zero, honours +`PropertyNameCaseInsensitive`, and throws `JsonException` for a wrong token or an +out-of-range number (a float beyond `float.MaxValue` is refused, not read as +infinity). Numbers follow `JsonSerializerOptions.NumberHandling`: NaN and the +infinities need `AllowNamedFloatingPointLiterals` (`"NaN"`, `"Infinity"`, +`"-Infinity"`). Every type is also a dictionary key: its flattened components +joined by commas, invariant culture (`"1,2"`, `"true,false"`, matrices row by +row, `rect` as `"minX,minY,maxX,maxY"`). A consumer resolver or converter +registered ahead of these wins. | Type | MessagePack | JSON | | --- | --- | --- | diff --git a/tests/GameCult.Caching.Tests/CultMathJsonTests.cs b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs index 64702b74..bf8444f9 100644 --- a/tests/GameCult.Caching.Tests/CultMathJsonTests.cs +++ b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs @@ -1,6 +1,9 @@ using System; +using System.Collections; +using System.Collections.Generic; using System.Linq; using System.Text.Json; +using System.Text.Json.Serialization; using CultMath; using FluentAssertions; using GameCult.Caching.MessagePack; @@ -99,6 +102,129 @@ namespace GameCult.Caching.Tests } } + private static JsonSerializerOptions With(JsonNumberHandling handling = JsonNumberHandling.Strict, bool ignoreCase = false) => + new JsonSerializerOptions { NumberHandling = handling, PropertyNameCaseInsensitive = ignoreCase }.AddCultMathConverters(); + + private const JsonNumberHandling Named = JsonNumberHandling.AllowNamedFloatingPointLiterals; + + [Test] + public void NonFiniteFloats_NeedNamedLiterals_AsInSystemTextJson() + { + Action nan = () => JsonSerializer.Serialize(new float2(float.NaN, 1f), Options); + nan.Should().Throw(); + Action inf = () => JsonSerializer.Serialize(new double3(1d, double.PositiveInfinity, 0d), Options); + inf.Should().Throw(); + Action named = () => JsonSerializer.Deserialize("""{"x":"NaN","y":1}""", Options); + named.Should().Throw(); + + var options = With(Named); + JsonSerializer.Serialize(new float2(float.NaN, 1f), options).Should().Be("""{"x":"NaN","y":1}"""); + JsonSerializer.Serialize(new float3(float.PositiveInfinity, float.NegativeInfinity, 0f), options) + .Should().Be("""{"x":"Infinity","y":"-Infinity","z":0}"""); + var back = JsonSerializer.Deserialize("""{"x":"Infinity","y":"-Infinity","z":"NaN"}""", options); + back.x.Should().Be(float.PositiveInfinity); + back.y.Should().Be(float.NegativeInfinity); + float.IsNaN(back.z).Should().BeTrue(); + JsonSerializer.Deserialize("""{"x":"NaN","y":"-Infinity"}""", options).y.Should().Be(double.NegativeInfinity); + } + + [Test] + public void Numbers_OutsideTheComponentType_AreRefused() + { + foreach (var json in new[] { """{"x":1e40}""", """{"x":-1e40}""" }) + { + Action tooBig = () => Parse(json); + tooBig.Should().Throw(json); + } + + Action doubleTooBig = () => Parse("""{"x":1e400}"""); + doubleTooBig.Should().Throw(); + Action alpha = () => Parse("""{"r":300}"""); + alpha.Should().Throw(); + Action negativeByte = () => Parse("""{"g":-1}"""); + negativeByte.Should().Throw(); + Action fraction = () => Parse("""{"x":1.5}"""); + fraction.Should().Throw(); + Action wrongToken = () => Parse("""{"x":"a"}"""); + wrongToken.Should().Throw(); + Action boolean = () => Parse("""{"x":1}"""); + boolean.Should().Throw(); + Action negativeState = () => Parse("""{"state":-1}"""); + negativeState.Should().Throw(); + } + + [Test] + public void NumberHandling_QuotesAndUnquotesNumbers() + { + JsonSerializer.Serialize(new float2(1f, 2.5f), With(JsonNumberHandling.WriteAsString)).Should().Be("""{"x":"1","y":"2.5"}"""); + JsonSerializer.Serialize(new int2(1, 2), With(JsonNumberHandling.WriteAsString)).Should().Be("""{"x":"1","y":"2"}"""); + JsonSerializer.Deserialize("""{"x":"1.5","y":2}""", With(JsonNumberHandling.AllowReadingFromString)) + .Should().Be(new float2(1.5f, 2f)); + JsonSerializer.Deserialize("""{"x":"7","y":2}""", With(JsonNumberHandling.AllowReadingFromString)) + .Should().Be(new int2(7, 2)); + Action strict = () => Parse("""{"x":"1.5"}"""); + strict.Should().Throw(); + Action tooBig = () => JsonSerializer.Deserialize("""{"x":"1e40"}""", With(JsonNumberHandling.AllowReadingFromString)); + tooBig.Should().Throw(); + } + + [Test] + public void PropertyNames_FollowTheCaseSensitivityOption() + { + JsonSerializer.Deserialize("""{"X":1,"Y":2}""", With(ignoreCase: true)).Should().Be(new float2(1f, 2f)); + JsonSerializer.Deserialize("""{"MIN":{"X":1,"y":2},"Max":{"x":3,"Y":4}}""", With(ignoreCase: true)) + .Should().Be(new rect(1f, 2f, 3f, 4f)); + Parse("""{"X":1,"Y":2}""").Should().Be(float2.zero); + } + + [Test] + public void EveryShape_WorksAsADictionaryKey() + { + Json(new Dictionary { [new int2(1, -2)] = "a" }).Should().Be("""{"1,-2":"a"}"""); + Parse>("""{"1,-2":"a"}""")[new int2(1, -2)].Should().Be("a"); + + KeyOf(new float2(1.5f, -2f)).Should().Be("1.5,-2"); + KeyOf(new float3(0.1f, 2f, 3f)).Should().Be("0.1,2,3"); + KeyOf(new bool2(true, false)).Should().Be("true,false"); + KeyOf(new Color32(255, 128, 0, 7)).Should().Be("255,128,0,7"); + KeyOf(new CultMath.Random(12345u)).Should().Be("12345"); + KeyOf(new float2x2(1f, 2f, 3f, 4f)).Should().Be("1,2,3,4"); + KeyOf(new rect(5f, 7f, -1f, -2f)).Should().Be("-1,-2,5,7"); + KeyOf(new float2(float.NaN, float.NegativeInfinity), Named).Should().Be("NaN,-Infinity"); + + var nan = JsonSerializer.Deserialize>("""{"NaN,Infinity":1}""", With(Named))!; + nan.Keys.Single().y.Should().Be(float.PositiveInfinity); + + Action refuseNonFinite = () => KeyOf(new float2(float.NaN, 0f)); + refuseNonFinite.Should().Throw(); + Action wrongArity = () => Parse>("""{"1,2":1}"""); + wrongArity.Should().Throw(); + Action overflow = () => Parse>("""{"1e40,0":1}"""); + overflow.Should().Throw(); + } + + [Test] + public void EveryPublicCultMathValueType_RoundTripsAsADictionaryKey() + { + foreach (var type in typeof(float2).Assembly.GetExportedTypes().Where(t => t.IsValueType && !t.IsEnum)) + { + var dictionary = (IDictionary)Activator.CreateInstance(typeof(Dictionary<,>).MakeGenericType(type, typeof(int)))!; + dictionary.Add(Activator.CreateInstance(type)!, 7); + + var json = JsonSerializer.Serialize(dictionary, dictionary.GetType(), Options); + var back = (IDictionary)JsonSerializer.Deserialize(json, dictionary.GetType(), Options)!; + + back.Count.Should().Be(1, type.Name); + back[Activator.CreateInstance(type)!].Should().Be(7, type.Name); + } + } + + private static string KeyOf(T value, JsonNumberHandling handling = JsonNumberHandling.Strict) where T : notnull + { + var json = JsonSerializer.Serialize(new Dictionary { [value] = 1 }, With(handling)); + return JsonDocument.Parse(json).RootElement.EnumerateObject().Single().Name; + } + private static void RoundTrip(T value) => Parse(Json(value)).Should().Be(value); } } From 26a4f4240bd180ffd2ec082166e115fae6bb6b0e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:29:02 +0200 Subject: [PATCH 19/64] Pin that a zero or missing Random state decodes to the generator's default seed Co-Authored-By: Claude Sonnet 5 --- .../CultMathSerializationTests.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs index fb6f1489..32531679 100644 --- a/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs +++ b/tests/GameCult.Caching.Tests/CultMathSerializationTests.cs @@ -141,6 +141,20 @@ namespace GameCult.Caching.Tests Decode("94CCFFCC800007").Should().Be(new Color32(255, 128, 0, 7)); } + + [Test] + public void Random_StateZero_IsTheGeneratorDefaultSeed() + { + // CultMath's generator maps state 0 to its default seed, so a zero or missing state is not degenerate. + var reference = new CultMath.Random(0u); + var expected = new[] { reference.NextUInt(), reference.NextUInt(), reference.NextUInt() }; + + foreach (var hex in new[] { "9100", "90" }) + { + var decoded = Decode(hex); + new[] { decoded.NextUInt(), decoded.NextUInt(), decoded.NextUInt() }.Should().Equal(expected); + } + } [Test] public void ConsumerResolvers_WinOverCultMathResolver() { From 7b7666f2c301bbad5daff6828ade7c802b3a2fc0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:38:20 +0200 Subject: [PATCH 20/64] cultnet-rs: drop and count packets that belong to no session A foreign connection id, a malformed frame, a sender no Connect admitted, a Connect with the peer table full, and any packet the session refuses ended the transport loop with an Err. Odin died twice this way after nginx reloads moved its UDP flows (require_connection: connection id does not match). The hub and the socket transport now drop such packets and count them in CultNetTransportStats::packets_dropped. Only socket I/O errors leave the loop. The socket transport checks the connection id before it touches the peer endpoint or session, so a stray Connect can no longer move the endpoint or reset the session. Co-Authored-By: Claude Sonnet 5 --- packages/cultnet-rs/src/rudp.rs | 55 +++- packages/cultnet-rs/src/transport.rs | 5 + packages/cultnet-rs/tests/rudp_server_hub.rs | 17 +- .../cultnet-rs/tests/rudp_stray_packets.rs | 255 ++++++++++++++++++ 4 files changed, 314 insertions(+), 18 deletions(-) create mode 100644 packages/cultnet-rs/tests/rudp_stray_packets.rs diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 174739be..226f5c7b 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -1572,8 +1572,15 @@ impl CultNetRudpServerHub { }; wire.truncate(received); self.stats.bytes_received += received as u64; - let packet = decode_rudp_packet(&wire)?; + // What a datagram carries is the sender's business, not a fault of this + // process: a malformed frame, another session's id and a sender no + // Connect admitted are dropped and counted. Only the socket ends a loop. + let Ok(packet) = decode_rudp_packet(&wire) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; if packet.connection_id != self.connection_id { + self.stats.packets_dropped += 1; return Ok(true); } @@ -1581,7 +1588,10 @@ impl CultNetRudpServerHub { if let Some(peer) = self.peers.get_mut(&remote_addr) && peer.context.connect_payload == packet.payload { - let _ = peer.session.receive(&packet, now_ms())?; + if peer.session.receive(&packet, now_ms()).is_err() { + self.stats.packets_dropped += 1; + return Ok(true); + } let reply = peer .session .pending_accept_for_resend(now_ms()) @@ -1590,7 +1600,8 @@ impl CultNetRudpServerHub { return Ok(true); } if !self.peers.contains_key(&remote_addr) && self.peers.len() >= self.max_peers { - return Err(anyhow!("RUDP server hub peer limit reached")); + self.stats.packets_dropped += 1; + return Ok(true); } let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: self.connection_id, @@ -1610,7 +1621,10 @@ impl CultNetRudpServerHub { session_generation: generation, connect_payload: packet.payload.clone(), }; - let accept = session.accept_connect(&packet, now_ms(), Vec::new())?; + let Ok(accept) = session.accept_connect(&packet, now_ms(), Vec::new()) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; self.send_packet(remote_addr, &accept)?; let replaced = self.peers.insert( remote_addr, @@ -1632,9 +1646,13 @@ impl CultNetRudpServerHub { } let Some(peer) = self.peers.get_mut(&remote_addr) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; + let Ok(result) = peer.session.receive(&packet, now_ms()) else { + self.stats.packets_dropped += 1; return Ok(true); }; - let result = peer.session.receive(&packet, now_ms())?; let context = peer.context.clone(); let ack = if packet.reliable { Some(peer.session.create_ack_for_received(packet.sequence)) @@ -1970,7 +1988,20 @@ impl CultNetRudpSocketTransportConnection { wire.truncate(received); self.stats.bytes_received += received as u64; - let packet = decode_rudp_packet(&wire)?; + // What a datagram carries is the sender's business, not a fault of this + // process: a malformed frame, another session's id, a sender that is not + // this transport's peer and a packet the session refuses are dropped and + // counted. Only the socket ends a loop. The id is checked before the + // peer endpoint or session state is touched, so a stray Connect can + // neither move the endpoint nor reset the session. + let Ok(packet) = decode_rudp_packet(&wire) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; + if packet.connection_id != self.session.connection_id() { + self.stats.packets_dropped += 1; + return Ok(true); + } if let Some(expected) = self.remote_addr { if expected != remote_addr { if self.mode == CultNetRudpSocketMode::Server @@ -1978,6 +2009,7 @@ impl CultNetRudpSocketTransportConnection { { self.remote_addr = Some(remote_addr); } else { + self.stats.packets_dropped += 1; return Ok(true); } } @@ -1985,6 +2017,7 @@ impl CultNetRudpSocketTransportConnection { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type != CultNetRudpPacketType::Connect { + self.stats.packets_dropped += 1; return Ok(true); } self.remote_addr = Some(remote_addr); @@ -1993,12 +2026,18 @@ impl CultNetRudpSocketTransportConnection { && packet.packet_type == CultNetRudpPacketType::Connect { self.session.reset_peer_state(); - let accept = self.session.accept_connect(&packet, now_ms(), Vec::new())?; + let Ok(accept) = self.session.accept_connect(&packet, now_ms(), Vec::new()) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; self.send_packet(&accept)?; return Ok(true); } - let result = self.session.receive(&packet, now_ms())?; + let Ok(result) = self.session.receive(&packet, now_ms()) else { + self.stats.packets_dropped += 1; + return Ok(true); + }; if let Some(reply) = result.reply { self.send_packet(&reply)?; } diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index d097aa3c..71e714c4 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -26,6 +26,11 @@ pub struct CultNetTransportStats { /// payload that lost a fragment on the wire. Zero for non-fragmenting /// transports. pub fragment_sets_evicted: u64, + /// Datagrams read and discarded because they belong to no session on this + /// transport: malformed frames, another session's connection id, an + /// unadmitted sender, or a packet the session refused. A steady rate means + /// a flow is being moved or scanned, not that this transport is failing. + pub packets_dropped: u64, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/packages/cultnet-rs/tests/rudp_server_hub.rs b/packages/cultnet-rs/tests/rudp_server_hub.rs index 201cf3b4..80cfde3b 100644 --- a/packages/cultnet-rs/tests/rudp_server_hub.rs +++ b/packages/cultnet-rs/tests/rudp_server_hub.rs @@ -415,16 +415,13 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { let mut second = client(server_addr)?; connect(&mut hub, &mut first, b"first")?; second.connect(b"second".to_vec())?; - for _ in 0..20 { - match hub.receive_event_once() { - Err(error) => { - assert!(error.to_string().contains("peer limit")); - return Ok(()); - } - Ok(_) => continue, - } - } - Err(anyhow!("hub did not enforce peer limit")) + // The peer limit refuses the second Connect without ending the hub loop: + // a full table is not a fault of the process that holds it. + std::thread::sleep(Duration::from_millis(20)); + while hub.receive_event_once()?.is_some() {} + assert_eq!(hub.sessions().len(), 1); + assert_eq!(hub.stats().packets_dropped, 1); + Ok(()) } #[test] diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs new file mode 100644 index 00000000..58f491fa --- /dev/null +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -0,0 +1,255 @@ +//! A packet that does not belong to a session is dropped; only socket failures +//! end a transport loop. Each test puts one class of stray in front of a real +//! session and requires the real session to keep being served. + +use std::net::{SocketAddr, UdpSocket}; +use std::time::Duration; + +use anyhow::Result; +use cultnet_rs::*; + +const CONNECTION_ID: u32 = 0x4355_4c54; +const FOREIGN_ID: u32 = 0x0BAD_F10E; + +#[derive(Clone, Copy)] +enum Stray { + /// Bytes that are not a RUDP frame. + Malformed, + /// A well-formed session that carries someone else's connection id: what a + /// moved UDP flow delivers. + ForeignConnectionId, + /// The right connection id from a sender no Connect ever admitted, or one + /// that already closed. + UnadmittedSender, +} + +fn socket() -> Result { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(20)))?; + Ok(socket) +} + +fn raw_session(connection_id: u32) -> CultNetRudpSession { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id, + initial_sequence: 1, + resend_delay_ms: 250, + max_pending_reliable_packets: None, + }) +} + +fn reliable() -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ..Default::default() + } +} + +fn send_to(socket: &UdpSocket, to: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + socket.send_to(&encode_rudp_packet(packet)?, to)?; + Ok(()) +} + +fn send_stray(kind: Stray, from: &UdpSocket, to: SocketAddr) -> Result<()> { + match kind { + Stray::Malformed => { + from.send_to(b"not a rudp packet", to)?; + from.send_to(&[], to)?; + } + Stray::ForeignConnectionId => { + let mut foreign = raw_session(FOREIGN_ID); + send_to(from, to, &foreign.create_connect(0, b"foreign".to_vec())?)?; + foreign.assume_connected(0); + send_to(from, to, &foreign.send("schema", b"foreign".to_vec(), reliable())?)?; + send_to(from, to, &foreign.create_ack())?; + send_to(from, to, &foreign.create_disconnect(b"foreign".to_vec()))?; + } + Stray::UnadmittedSender => { + let mut orphan = raw_session(CONNECTION_ID); + orphan.assume_connected(0); + send_to(from, to, &orphan.send("schema", b"orphan".to_vec(), reliable())?)?; + send_to(from, to, &orphan.create_ack())?; + send_to(from, to, &orphan.create_disconnect(b"orphan".to_vec()))?; + } + } + Ok(()) +} + +fn hub_keeps_serving_its_session(kind: Stray) -> Result<()> { + let mut options = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); + options.max_peers = 1; + let mut hub = CultNetRudpServerHub::new(options)?; + let hub_addr = hub.local_addr()?; + let mut real = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("real", socket()?, hub_addr, CONNECTION_ID), + )?; + real.connect(b"real".to_vec())?; + let mut admitted = false; + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Connected { .. }) = hub.receive_event_once()? { + admitted = true; + break; + } + } + assert!(admitted); + let _ = real.receive_once()?; + + let stray = socket()?; + send_stray(kind, &stray, hub_addr)?; + std::thread::sleep(Duration::from_millis(20)); + while hub.receive_event_once()?.is_some() {} + assert!(hub.stats().packets_dropped > 0, "strays are counted"); + + real.send("schema", b"still here".to_vec())?; + std::thread::sleep(Duration::from_millis(20)); + match hub.receive_event_once()? { + Some(CultNetRudpServerEvent::Frame { frame, .. }) => { + assert_eq!(frame.payload, b"still here"); + } + other => panic!("the real session must still be served, got {other:?}"), + } + Ok(()) +} + +#[test] +fn hub_drops_a_connect_that_arrives_with_the_peer_table_full() -> Result<()> { + let mut options = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); + options.max_peers = 1; + let mut hub = CultNetRudpServerHub::new(options)?; + let hub_addr = hub.local_addr()?; + let mut real = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("real", socket()?, hub_addr, CONNECTION_ID), + )?; + real.connect(b"real".to_vec())?; + while !matches!( + hub.receive_event_once()?, + Some(CultNetRudpServerEvent::Connected { .. }) + ) {} + let _ = real.receive_once()?; + + send_to( + &socket()?, + hub_addr, + &raw_session(CONNECTION_ID).create_connect(0, b"late".to_vec())?, + )?; + std::thread::sleep(Duration::from_millis(20)); + while hub.receive_event_once()?.is_some() {} + assert_eq!(hub.sessions().len(), 1); + assert_eq!(hub.stats().packets_dropped, 1); + + real.send("schema", b"still here".to_vec())?; + std::thread::sleep(Duration::from_millis(20)); + assert!(matches!( + hub.receive_event_once()?, + Some(CultNetRudpServerEvent::Frame { .. }) + )); + Ok(()) +} + +/// A moved flow answers a client from its server address with someone else's +/// session. +fn client_keeps_its_session(kind: Stray) -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server.recv_from(&mut buffer)?; + let connect = decode_rudp_packet(&buffer[..received])?; + let mut server_session = raw_session(CONNECTION_ID); + let accept = server_session.accept_connect(&connect, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + assert!(client.connected()); + + send_stray(kind, &server, client_addr)?; + send_to( + &server, + client_addr, + &server_session.send("schema", b"real".to_vec(), reliable())?, + )?; + let frame = client + .receive_once()? + .expect("the real frame behind the strays"); + assert_eq!(frame.payload, b"real"); + assert!(client.stats().packets_dropped > 0, "strays are counted"); + Ok(()) +} + +/// A server-mode transport owns one peer. A stray, including a foreign Connect +/// from another address, neither resets that peer's session nor moves the +/// endpoint the transport answers. +fn server_mode_keeps_its_peer(kind: Stray) -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &peer_session.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + peer_session.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + + send_stray(kind, &socket()?, server_addr)?; + send_stray(kind, &peer, server_addr)?; + send_to( + &peer, + server_addr, + &peer_session.send("schema", b"real".to_vec(), reliable())?, + )?; + let frame = server + .receive_once()? + .expect("the real frame behind the strays"); + assert_eq!(frame.payload, b"real"); + assert!(server.stats().packets_dropped > 0, "strays are counted"); + Ok(()) +} + +macro_rules! per_stray { + ($body:ident, $malformed:ident, $foreign:ident, $unadmitted:ident) => { + #[test] + fn $malformed() -> Result<()> { + $body(Stray::Malformed) + } + #[test] + fn $foreign() -> Result<()> { + $body(Stray::ForeignConnectionId) + } + #[test] + fn $unadmitted() -> Result<()> { + $body(Stray::UnadmittedSender) + } + }; + ($body:ident, $malformed:ident, $foreign:ident) => { + #[test] + fn $malformed() -> Result<()> { + $body(Stray::Malformed) + } + #[test] + fn $foreign() -> Result<()> { + $body(Stray::ForeignConnectionId) + } + }; +} + +per_stray!( + hub_keeps_serving_its_session, + hub_survives_malformed_frames, + hub_survives_foreign_connection_ids, + hub_survives_unadmitted_senders +); +per_stray!( + client_keeps_its_session, + client_survives_malformed_frames, + client_survives_foreign_connection_ids +); +per_stray!( + server_mode_keeps_its_peer, + server_mode_survives_malformed_frames, + server_mode_survives_foreign_connection_ids +); From ba8466114d39f25f932316b534d63bfd1323edb9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:38:20 +0200 Subject: [PATCH 21/64] cultmesh-rs: RUDP document server drops refused Connects instead of dying A Connect storm from a flow that never hears its Accept filled the session reliable queue and accept_connect returned Err out of poll_once. A refused Connect, a malformed frame and a packet for an unknown session are now counted in packets_dropped() and poll_once keeps serving. Co-Authored-By: Claude Sonnet 5 --- .../cultmesh-rs/src/rudp_document_server.rs | 35 +++++++++++--- .../cultmesh-rs/tests/rudp_document_server.rs | 48 +++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index bebeb6c3..d5e0b473 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -207,6 +207,7 @@ pub struct CultMeshRudpDocumentServer { snapshot_source: Q, clock: C, options: CultMeshRudpDocumentServerOptions, + packets_dropped: u64, } impl CultMeshRudpDocumentServer @@ -234,9 +235,18 @@ where snapshot_source, clock, options, + packets_dropped: 0, }) } + /// Datagrams read and discarded because they belong to no admissible + /// session: malformed frames, unknown sessions, a Connect the session or + /// the session cap refuses, a packet that fails its session. Never an + /// error: a moved flow or a scanner must not end the daemon loop. + pub fn packets_dropped(&self) -> u64 { + self.packets_dropped + } + pub fn local_addr(&self) -> Result { Ok(self.socket.local_addr()?) } @@ -264,7 +274,10 @@ where let packet = match decode_rudp_packet(&wire) { Ok(packet) => packet, - Err(_) => return Ok(CultMeshRudpPollOutcome::Handled), + Err(_) => { + self.packets_dropped += 1; + return Ok(CultMeshRudpPollOutcome::Handled); + } }; let now_unix = self.clock.now_unix_millis(); let now_monotonic = self.clock.now_monotonic_millis(); @@ -274,11 +287,14 @@ where }; if packet.packet_type == CultNetRudpPacketType::Connect { - self.accept_connection(key, &packet, now_monotonic)?; + if !self.accept_connection(key, &packet, now_monotonic)? { + self.packets_dropped += 1; + } return Ok(CultMeshRudpPollOutcome::Handled); } if !self.sessions.contains_key(&key) { + self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } if packet.packet_type == CultNetRudpPacketType::Data @@ -310,6 +326,7 @@ where Ok(result) => result, Err(_) => { self.sessions.remove(&key); + self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } }; @@ -385,15 +402,17 @@ where }) } + /// `Ok(false)` when the Connect was refused (session cap, or the session + /// cannot queue another Accept); only a socket failure is an `Err`. fn accept_connection( &mut self, key: CultMeshRudpSessionKey, packet: &CultNetRudpPacket, now: u64, - ) -> Result<()> { + ) -> Result { if !self.sessions.contains_key(&key) { if self.sessions.len() >= self.options.max_sessions { - return Ok(()); + return Ok(false); } self.sessions.insert( key, @@ -421,9 +440,13 @@ where // The same key is a retransmitted Connect for the existing epoch. // A genuinely fresh client incarnation must choose a fresh id. entry.last_activity_monotonic_millis = now; - entry.session.accept_connect(packet, now, Vec::new())? + match entry.session.accept_connect(packet, now, Vec::new()) { + Ok(accept) => accept, + Err(_) => return Ok(false), + } }; - self.send_packet(key.remote_addr, &accept) + self.send_packet(key.remote_addr, &accept)?; + Ok(true) } fn deliver_application_message( diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index df14b4ea..8bf816ae 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -553,3 +553,51 @@ fn session_cap_and_expiry_use_monotonic_time() -> Result<()> { assert_eq!(server.session_count(), 0); Ok(()) } + +#[test] +fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { + let clock = Clock::new(60_000); + let sink = Sink::default(); + let options = CultMeshRudpDocumentServerOptions { + max_pending_reliable_packets_per_session: 4, + ..Default::default() + }; + let mut server = server(options, clock, sink.clone(), Source::default())?; + let target = server.local_addr()?; + let mut real = client(target, 101)?; + connect(&mut server, &mut [&mut real])?; + + // A moved flow re-sends Connect from a socket that never hears the Accept: + // each one asks the session to queue one more unacknowledged Accept. + let storm = UdpSocket::bind("127.0.0.1:0")?; + let mut storm_client = cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { + connection_id: 102, + initial_sequence: 1, + resend_delay_ms: 10, + max_pending_reliable_packets: None, + }); + let connect_packet = storm_client.create_connect(0, Vec::new())?; + for _ in 0..12 { + storm.send_to(&cultnet_rs::encode_rudp_packet(&connect_packet)?, target)?; + server.poll_once()?; + } + storm.send_to(b"not a rudp packet", target)?; + server.poll_once()?; + + send( + &mut real, + &CultNetMessage::DocumentPutRaw { + message_id: "after-storm".into(), + document: document("real", vec![9]), + }, + )?; + for _ in 0..20 { + server.poll_once()?; + if sink.0.lock().unwrap().receipts.len() == 1 { + break; + } + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + assert!(server.packets_dropped() > 0, "the storm and the stray are counted"); + Ok(()) +} From ecb80b4fd8e838725efc10b991c3d68d45fd7bfb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:44:37 +0200 Subject: [PATCH 22/64] cultnet-ts: RUDP socket transport drops and counts strays instead of emitting error A foreign connection id, a malformed frame or a refused packet was emitted as an error event; with no listener that throws and ends the process. They are now counted in stats.packetsDropped. The id is checked before the peer endpoint or session state is touched, and an unadmitted first packet in server mode no longer claims the endpoint. Listener faults still surface as error events. Co-Authored-By: Claude Sonnet 5 --- packages/cultnet-ts/src/rudp.ts | 43 +++++++++++++--- packages/cultnet-ts/src/transport.ts | 3 ++ packages/cultnet-ts/test/cultnet.test.ts | 63 ++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 56dc67fa..89e478c2 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -807,6 +807,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement bytesSent: 0, framesReceived: 0, framesSent: 0, + packetsDropped: 0, }; constructor(options: CultNetRudpSocketTransportOptions) { @@ -922,15 +923,29 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement #receiveDatagram(wire: Buffer, remote: RemoteInfo): void { this.#stats.bytesReceived += wire.length; + // What a datagram carries is the sender's business, not a fault of this + // process: a malformed frame, another session's connection id and a packet + // the session refuses are dropped and counted, never emitted as "error" + // (an "error" with no listener throws and ends the process). The id is + // checked before the peer endpoint or session state is touched, so a stray + // Connect can neither move the endpoint nor reset the session. let packet: CultNetRudpPacket; try { packet = decodeRudpPacket(wire); - } catch (error) { - this.emit("error", error instanceof Error ? error : new Error(String(error))); + } catch { + this.#stats.packetsDropped += 1; + return; + } + if (packet.connectionId !== this.#session.connectionId) { + this.#stats.packetsDropped += 1; return; } if (!this.#remoteHost || this.#remotePort === undefined) { + if (this.#mode === "server" && packet.packetType !== "connect") { + this.#stats.packetsDropped += 1; + return; + } this.#remoteHost = remote.address; this.#remotePort = remote.port; } else if (remote.address !== this.#remoteHost || remote.port !== this.#remotePort) { @@ -938,18 +953,32 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#remoteHost = remote.address; this.#remotePort = remote.port; } else { + this.#stats.packetsDropped += 1; return; } } - try { - if (this.#mode === "server" && packet.packetType === "connect") { - this.#session.resetPeerState(); - this.#sendPacket(this.#session.acceptConnect(packet, Date.now())); + if (this.#mode === "server" && packet.packetType === "connect") { + this.#session.resetPeerState(); + let accept: CultNetRudpPacket; + try { + accept = this.#session.acceptConnect(packet, Date.now()); + } catch { + this.#stats.packetsDropped += 1; return; } + this.#sendPacket(accept); + return; + } - const result = this.#session.receive(packet, Date.now()); + let result: ReturnType; + try { + result = this.#session.receive(packet, Date.now()); + } catch { + this.#stats.packetsDropped += 1; + return; + } + try { if (result.reply) { this.#sendPacket(result.reply); } diff --git a/packages/cultnet-ts/src/transport.ts b/packages/cultnet-ts/src/transport.ts index a1b9b3ce..f900aa23 100644 --- a/packages/cultnet-ts/src/transport.ts +++ b/packages/cultnet-ts/src/transport.ts @@ -14,6 +14,8 @@ export interface CultNetTransportStats { bytesSent: number; framesReceived: number; framesSent: number; + /** Datagrams read and discarded because they belong to no session on this transport. */ + packetsDropped: number; } export interface CultNetReconnectPolicy { @@ -192,6 +194,7 @@ export class TcpFramedTransportConnection extends EventEmitter implements CultNe bytesSent: 0, framesReceived: 0, framesSent: 0, + packetsDropped: 0, }; constructor(stream: Duplex, profile: CultNetTransportProfile) { diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 2f540a86..735418e0 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -1867,3 +1867,66 @@ test("rudp receiver evicts stranded fragment sets instead of refusing new ones", assert.equal(second.delivered.length, 1); assert.equal(Buffer.from(second.delivered[0]!.payload).toString(), "hello"); }); + +test("rudp socket transport drops strays without an error event and keeps serving its peer", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const strayHost = await bindUdpSocket(); + const connectionId = 0x10203050; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 5, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendPollMs: 5, + }); + + try { + // No "error" listener on either side: an unhandled "error" event throws + // and would end the process, which is what a stray must never do. + const frames: string[] = []; + server.on("frame", (frame) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + client.connect(); + await waitFor(() => client.connected && server.connected, "RUDP socket handshake"); + + const foreign = new CultNetRudpSession({ connectionId: 0x0badf10e }); + const strays = [ + Buffer.from("not a rudp packet"), + Buffer.alloc(0), + Buffer.from(encodeRudpPacket(foreign.createConnect(0, Buffer.from("foreign")))), + ]; + for (const stray of strays) { + // From an unrelated address, and from the peer's own address. + strayHost.send(stray, udpPort(serverSocket), "127.0.0.1"); + clientSocket.send(stray, udpPort(serverSocket), "127.0.0.1"); + } + await waitFor(() => server.stats.packetsDropped >= strays.length * 2, "strays counted"); + + client.send("schema", Buffer.from("still here", "utf8")); + await waitFor(() => frames.length === 1, "the real frame"); + assert.deepEqual(frames, ["still here"]); + + // The client side too: a moved flow answers from the server address. + for (const stray of strays) { + serverSocket.send(stray, udpPort(clientSocket), "127.0.0.1"); + } + await waitFor(() => client.stats.packetsDropped >= strays.length, "client strays counted"); + const clientFrames: string[] = []; + client.on("frame", (frame) => clientFrames.push(Buffer.from(frame.payload).toString("utf8"))); + server.send("schema", Buffer.from("server still here", "utf8")); + await waitFor(() => clientFrames.length === 1, "the real server frame"); + } finally { + strayHost.close(); + client.close(); + server.close(); + } +}); From 0a543c9fb29a4773398d8f32686fb0ba2f27f533 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:46:36 +0200 Subject: [PATCH 23/64] Kill the Stryker survivors in the projection and observer cuts Delete the dead removed-ancestor branch and the duplicated name-read in the variant name rule (resolution already refuses a missing or removed base and a variant shares its base type). Add tests for the index rule counting a record the same commit removes, blank names not colliding, and Dispose rethrowing escaped completions after every observer completed. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 16 +++---- .../CultCacheObserverTests.cs | 35 ++++++++++++++++ .../CultCacheProjectionTests.cs | 42 +++++++++++++++++++ 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index d2115535..f9497cb3 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2551,7 +2551,7 @@ namespace GameCult.Caching var descriptor = stored.Descriptor; string? name = null; if (descriptor.NameAccessor != null) - name = Read(stored, $"[CultName] member {descriptor.NameMember}", descriptor.NameAccessor) is { Length: > 0 } read ? read : null; + name = NameRead(stored) is { Length: > 0 } read ? read : null; var indexes = new List<(string Alias, string Value)>(); foreach (var pair in descriptor.IndexAccessors) { @@ -2565,6 +2565,9 @@ namespace GameCult.Caching } } + private static string? NameRead(CultStoredDocument stored) => + Read(stored, $"[CultName] member {stored.Descriptor.NameMember}", stored.Descriptor.NameAccessor!); + private static string? Read(CultStoredDocument stored, string what, Func getter) { try @@ -2590,7 +2593,7 @@ namespace GameCult.Caching // The variant rules are judged only when a variant is involved. if (_variantKeys.Count != 0 || admitted.Any(stored => stored.Variant != null)) { - RefuseVariantNameSharing(plan, evicted); + RefuseVariantNameSharing(plan); RefuseVariantIndexSharing(plan, evicted); } @@ -2744,20 +2747,17 @@ namespace GameCult.Caching // R6: the name index maps one name to one key, so a variant that kept its base's name would shadow it. A variant's // name is its projection; an ancestor that lands is read from its projection, one that stays is read as held. - private void RefuseVariantNameSharing(VariantPlan plan, IReadOnlyList evicted) + private void RefuseVariantNameSharing(VariantPlan plan) { - var gone = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); var landing = plan.Admitted.Concat(plan.Dependents).ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); + // Resolution already refused a base that is missing or removed, so an ancestor is landing or held. CultStoredDocument? Ancestor(string key) => landing.TryGetValue(key, out var landed) ? landed - : gone.Contains(key) ? null : _entries.TryGetValue(key, out var held) ? held : null; string? NameOf(CultStoredDocument stored) => - plan.Projections.TryGetValue(stored.Key.Value, out var projection) - ? projection.Name - : stored.Descriptor.NameAccessor == null ? null : Read(stored, $"[CultName] member {stored.Descriptor.NameMember}", stored.Descriptor.NameAccessor); + plan.Projections.TryGetValue(stored.Key.Value, out var projection) ? projection.Name : NameRead(stored); foreach (var variant in landing.Values.Where(stored => stored.Variant != null)) { diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs index 8cb3992d..7fde01e7 100644 --- a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -135,5 +135,40 @@ namespace GameCult.Caching.Tests using var after = cache.Watch().Subscribe(_ => { }, _ => completed++); Assert.That(completed, Is.EqualTo(2), "subscribing to a disposed cache completes at once"); } + + // Completing an observer is publication too: every observer is completed, and a fail-fast handler's exception + // reaches the disposer afterwards (an AggregateException for several). + [Test] + public void ACompletionThatEscapesReachesTheDisposerAfterEveryObserverCompleted() + { + var cache = new CultCache(Registry); + var completed = new List(); + using var one = cache.Watch().Subscribe(_ => { }, _ => + { + completed.Add("one"); + throw new InvalidOperationException("one"); + }); + using var two = cache.Watch().Subscribe(_ => { }, _ => + { + completed.Add("two"); + throw new InvalidOperationException("two"); + }); + + var escaped = Assert.Throws(() => cache.Dispose())!; + + Assert.That(completed, Is.EqualTo(new[] { "one", "two" })); + Assert.That(escaped.InnerExceptions.Select(exception => exception.Message), Is.EqualTo(new[] { "one", "two" })); + } + + [Test] + public void ASingleEscapedCompletionIsRethrownAsItself() + { + var cache = new CultCache(Registry); + using var only = cache.Watch().Subscribe(_ => { }, _ => throw new InvalidOperationException("only")); + + var escaped = Assert.Throws(() => cache.Dispose())!; + + Assert.That(escaped.Message, Is.EqualTo("only")); + } } } diff --git a/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs index 88f251c6..57135929 100644 --- a/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheProjectionTests.cs @@ -212,5 +212,47 @@ namespace GameCult.Caching.Tests Assert.That(cache.GetByIndex("code", "base-code-2"), Is.Not.Null); Assert.That(cache.GetByName("variant"), Is.Not.Null); } + + // The index rule counts the holders a commit leaves behind: a record removed by the same commit no longer holds its value. + [Test] + public void AVariantMayTakeAnIndexedValueFromARecordTheSameCommitRemoves() + { + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + var baseKey = new CultRecordKey("projected:base"); + var holderKey = new CultRecordKey("projected:holder"); + Put(cache, new ProjectedRecord { Name = "base", Code = "base-code" }, baseKey); + Put(cache, new ProjectedRecord { Name = "holder", Code = "shared" }, holderKey); + + var landed = cache.Commit(batch => + { + batch.Remove(holderKey); + batch.UpsertVariant(new CultRecordKey("projected:variant"), baseKey, new[] + { + cache.Override(nameof(ProjectedRecord.Name), "variant"), + cache.Override(nameof(ProjectedRecord.Code), "shared") + }); + }); + + Assert.That(landed, Is.True); + Assert.That(cache.GetByIndex("code", "shared"), Is.Not.Null); + Assert.That(cache.Get(holderKey), Is.Null); + } + + // A blank name is no name: a variant and its base that are both blank do not shadow each other. + [Test] + public void ABlankNameIsNoNameSoBlankVariantAndBaseDoNotCollide() + { + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + var baseKey = new CultRecordKey("projected:base"); + Put(cache, new ProjectedRecord { Name = "", Code = "base-code" }, baseKey); + + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("projected:variant"), baseKey, new[] + { + cache.Override(nameof(ProjectedRecord.Name), ""), + cache.Override(nameof(ProjectedRecord.Code), "variant-code") + })); + + Assert.That(cache.Get(new CultRecordKey("projected:variant")), Is.Not.Null); + } } } From dbe4f8a8503ae76d8e2ed4bcc49a22fe3c5ad80b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:48:10 +0200 Subject: [PATCH 24/64] cultnet-py: RUDP socket transport drops and counts strays A malformed frame, a foreign connection id or a refused packet raised out of receive_once. They are now counted in stats.packets_dropped and receive_once returns None. The id is checked before the peer endpoint is claimed, so a stray arriving first can no longer become the server-mode peer. Co-Authored-By: Claude Sonnet 5 --- .../cultnet-py/src/cultnet_py/transport.py | 34 ++++++++++- packages/cultnet-py/tests/test_cultnet.py | 59 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 73bd1e62..14c33fcb 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -23,6 +23,10 @@ class CultNetTransportStats: bytes_sent: int = 0 frames_received: int = 0 frames_sent: int = 0 + # Datagrams read and discarded because they belong to no session on this + # transport: malformed frames, another session's connection id, a sender + # that is not the peer, or a packet the session refuses. + packets_dropped: int = 0 @dataclass(frozen=True) @@ -921,6 +925,7 @@ class CultNetRudpSocketTransportConnection: self._bytes_sent = 0 self._frames_received = 0 self._frames_sent = 0 + self._packets_dropped = 0 self._delivered_frames: deque[CultNetTransportFrame] = deque() self._closed = False self.disconnect_reason: bytes | None = None @@ -943,6 +948,7 @@ class CultNetRudpSocketTransportConnection: bytes_sent=self._bytes_sent, frames_received=self._frames_received, frames_sent=self._frames_sent, + packets_dropped=self._packets_dropped, ) @property @@ -977,17 +983,39 @@ class CultNetRudpSocketTransportConnection: return None self._bytes_received += len(wire) + # What a datagram carries is the sender's business, not a fault of this + # process: a malformed frame, another session's connection id, a sender + # that is not the peer and a packet the session refuses are dropped and + # counted. Only the socket ends a loop. The id is checked before the peer + # endpoint is claimed, so a stray cannot become the peer. + try: + packet = decode_rudp_packet(wire) + except ValueError: + self._packets_dropped += 1 + return None + if packet.connection_id != self.session.connection_id: + self._packets_dropped += 1 + return None if self.remote_addr is None: self.remote_addr = remote_addr elif remote_addr != self.remote_addr: + self._packets_dropped += 1 return None - packet = decode_rudp_packet(wire) if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: - self._send_packet(self.session.accept_connect(packet, _now_ms())) + try: + accept = self.session.accept_connect(packet, _now_ms()) + except ValueError: + self._packets_dropped += 1 + return None + self._send_packet(accept) return None - result = self.session.receive(packet, _now_ms()) + try: + result = self.session.receive(packet, _now_ms()) + except ValueError: + self._packets_dropped += 1 + return None if result.reply is not None: self._send_packet(result.reply) for ready in result.ready_to_send: diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 475305d3..549d0803 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -918,6 +918,65 @@ class CultNetTests(unittest.TestCase): client.close() server.close() + def test_cultnet_rudp_socket_transport_drops_strays_and_keeps_serving_its_peer(self) -> None: + server_socket = bind_udp_socket() + client_socket = bind_udp_socket() + stray_socket = bind_udp_socket() + connection_id = 0x10203050 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + ) + ) + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-client", + socket=client_socket, + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=server_socket.getsockname(), + connection_id=connection_id, + ) + ) + foreign = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=0x0BADF10E)) + strays = [ + b"not a rudp packet", + b"", + encode_rudp_packet(foreign.create_connect(0, b"foreign")), + ] + + try: + # A stray that arrives before the peer must not become the peer. + for stray in strays: + stray_socket.sendto(stray, server_socket.getsockname()) + for _ in strays: + self.assertIsNone(server.receive_once()) + self.assertEqual(server.stats.packets_dropped, len(strays)) + + client.connect(b"join") + pump_rudp_handshake(client, server) + + # A moved flow answers from the peer's own address with someone else's session. + for stray in strays: + server_socket.sendto(stray, client_socket.getsockname()) + client_socket.sendto(stray, server_socket.getsockname()) + for _ in strays: + self.assertIsNone(client.receive_once()) + self.assertIsNone(server.receive_once()) + self.assertEqual(client.stats.packets_dropped, len(strays)) + self.assertEqual(server.stats.packets_dropped, 2 * len(strays)) + + client.send("schema", b"still here") + self.assertEqual(receive_rudp_frame(server).payload, b"still here") + server.send("schema", b"server still here") + self.assertEqual(receive_rudp_frame(client).payload, b"server still here") + finally: + stray_socket.close() + client.close() + server.close() + def test_cultnet_rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames(self) -> None: server_socket = bind_udp_socket() client_socket = bind_udp_socket() From b8dbcfe40c05f9c4ed533d3c1b2cb26a7424924c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:51:32 +0200 Subject: [PATCH 25/64] GameCult.Networking: RUDP socket transports drop and count strays A malformed frame, a foreign connection id or a packet the session refuses threw out of TryReceiveOnce. They are now counted in Stats.PacketsDropped and the read reports progress. The client-side transport checks the connection id before it claims the peer endpoint; the server listener drops refused packets instead of throwing. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Networking/CultNetTransport.cs | 74 +++++++++++++++++-- .../NetworkingTests.cs | 62 ++++++++++++++++ 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index f549f735..9df650cd 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -33,6 +33,12 @@ namespace GameCult.Networking /// Gets the number of bytes received, including transport framing. /// public long BytesReceived { get; internal set; } + /// + /// Gets the number of datagrams read and discarded because they belong to no session on + /// this transport: malformed frames, another session's connection id, a sender that is not + /// the peer, or a packet the session refuses. + /// + public long PacketsDropped { get; internal set; } internal CultNetTransportStats Snapshot() { @@ -41,7 +47,8 @@ namespace GameCult.Networking FramesSent = FramesSent, FramesReceived = FramesReceived, BytesSent = BytesSent, - BytesReceived = BytesReceived + BytesReceived = BytesReceived, + PacketsDropped = PacketsDropped }; } } @@ -2081,23 +2088,55 @@ namespace GameCult.Networking } _stats.BytesReceived += received; + + // What a datagram carries is the sender's business, not a fault of this process: a + // malformed frame, another session's connection id, a sender that is not the peer and + // a packet the session refuses are dropped and counted. Only the socket ends a loop. + // The id is checked before the peer endpoint is claimed, so a stray cannot become it. + var wire = new byte[received]; + Array.Copy(buffer, wire, received); + CultNetRudpPacket packet; + try + { + packet = CultNetRudpPacketCodec.Decode(wire); + } + catch (Exception error) when (error is InvalidOperationException || error is ArgumentException) + { + _stats.PacketsDropped++; + return true; + } + if (packet.ConnectionId != _session.ConnectionId) + { + _stats.PacketsDropped++; + return true; + } if (_remoteEndPoint == null) { _remoteEndPoint = remote; } else if (!_remoteEndPoint.Equals(remote)) { + _stats.PacketsDropped++; return true; } - var wire = new byte[received]; - Array.Copy(buffer, wire, received); - var packet = CultNetRudpPacketCodec.Decode(wire); TracePacket("rx", packet, received, remote); if (_mode == CultNetRudpSocketMode.Server && packet.PacketType == CultNetRudpPacketType.Connect) { lock (_sessionGate) - SendPacket(_session.AcceptConnect(packet, NowMs())); + { + CultNetRudpPacket accept; + try + { + accept = _session.AcceptConnect(packet, NowMs()); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } + SendPacket(accept); + } return true; } @@ -2105,7 +2144,15 @@ namespace GameCult.Networking CultNetRudpPacket? acknowledgement = null; lock (_sessionGate) { - result = _session.Receive(packet, NowMs()); + try + { + result = _session.Receive(packet, NowMs()); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } if (result.Reply != null) SendPacket(result.Reply); SendPackets(result.ReadyToSend); @@ -2478,13 +2525,15 @@ namespace GameCult.Networking { packet = CultNetRudpPacketCodec.Decode(wire); } - catch (InvalidOperationException) + catch (Exception error) when (error is InvalidOperationException || error is ArgumentException) { + _stats.PacketsDropped++; return true; } if (packet.ConnectionId != _connectionId) { + _stats.PacketsDropped++; return true; } @@ -2508,6 +2557,7 @@ namespace GameCult.Networking if (!_peers.TryGetValue(peerKey, out var existingPeer)) { + _stats.PacketsDropped++; return true; } @@ -2515,7 +2565,15 @@ namespace GameCult.Networking CultNetRudpPacket? acknowledgement = null; lock (existingPeer.SessionGate) { - result = existingPeer.Session.Receive(packet, NowMs()); + try + { + result = existingPeer.Session.Receive(packet, NowMs()); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } if (result.Reply != null) SendPacket(existingPeer.RemoteEndPoint, result.Reply); SendPackets(existingPeer, result.ReadyToSend); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 40bf43eb..d94b877e 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1193,6 +1193,68 @@ namespace GameCult.Networking.Tests } + [Test] + public void RudpSocketTransport_DropsStraysAndKeepsServingItsPeer() + { + using var serverSocket = BindUdpSocket(); + using var clientSocket = BindUdpSocket(); + using var strayHost = BindUdpSocket(); + var serverEndPoint = serverSocket.LocalEndPoint!; + var clientEndPoint = clientSocket.LocalEndPoint!; + const uint connectionId = 0x10203050; + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = connectionId + }); + using var client = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = clientSocket, + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = serverEndPoint, + ConnectionId = connectionId + }); + var foreign = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 0x0BADF10E }); + var strays = new[] + { + Encoding.UTF8.GetBytes("not a rudp packet"), + new byte[] { 1 }, + CultNetRudpPacketCodec.Encode(foreign.CreateConnect(0, Encoding.UTF8.GetBytes("foreign"))) + }; + + // A stray that arrives before the peer must not become the peer. + foreach (var stray in strays) + strayHost.SendTo(stray, serverEndPoint); + for (var index = 0; index < strays.Length; index++) + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(strays.Length)); + + client.Connect(Encoding.UTF8.GetBytes("join")); + PumpRudpHandshake(client, server); + + // A moved flow answers from the peer's own address with someone else's session. + foreach (var stray in strays) + { + serverSocket.SendTo(stray, clientEndPoint); + clientSocket.SendTo(stray, serverEndPoint); + } + for (var index = 0; index < strays.Length; index++) + { + Assert.That(client.ReceiveOnce(), Is.Null); + Assert.That(server.ReceiveOnce(), Is.Null); + } + Assert.That(client.Stats.PacketsDropped, Is.EqualTo(strays.Length)); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(2 * strays.Length)); + + client.Send("schema", Encoding.UTF8.GetBytes("still here")); + Assert.That(Encoding.UTF8.GetString(ReceiveRudpFrame(server).Payload), Is.EqualTo("still here")); + server.Send("schema", Encoding.UTF8.GetBytes("server still here")); + Assert.That(Encoding.UTF8.GetString(ReceiveRudpFrame(client).Payload), Is.EqualTo("server still here")); + } + [Test] public void RudpSocketTransport_CarriesCultNetSchemaMessages() { From 6b3da283b4b0cd2223050631e238c1d86bbd22a1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:12:09 +0200 Subject: [PATCH 26/64] cultmesh-rs: pin each RUDP document server drop site with an exact count Co-Authored-By: Claude Sonnet 5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 8bf816ae..e1e912e0 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -556,6 +556,10 @@ fn session_cap_and_expiry_use_monotonic_time() -> Result<()> { #[test] fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { + use cultnet_rs::{ + CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, + encode_rudp_packet, + }; let clock = Clock::new(60_000); let sink = Sink::default(); let options = CultMeshRudpDocumentServerOptions { @@ -566,23 +570,57 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { let target = server.local_addr()?; let mut real = client(target, 101)?; connect(&mut server, &mut [&mut real])?; + assert_eq!(server.packets_dropped(), 0); + + let raw = UdpSocket::bind("127.0.0.1:0")?; + let session = |id| { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: id, + initial_sequence: 1, + resend_delay_ms: 10, + max_pending_reliable_packets: None, + }) + }; + let mut send_raw = |server: &mut Server, packet: &cultnet_rs::CultNetRudpPacket| -> Result<()> { + raw.send_to(&encode_rudp_packet(packet)?, target)?; + server.poll_once()?; + Ok(()) + }; + + // A malformed frame. + raw.send_to(b"not a rudp packet", target)?; + server.poll_once()?; + assert_eq!(server.packets_dropped(), 1); + + // Data for a session no Connect admitted. + let mut unadmitted = session(300); + unadmitted.assume_connected(0); + let reliable = CultNetRudpSendOptions { + reliable: true, + ..Default::default() + }; + send_raw(&mut server, &unadmitted.send("schema", vec![1], reliable.clone())?)?; + assert_eq!(server.packets_dropped(), 2); // A moved flow re-sends Connect from a socket that never hears the Accept: - // each one asks the session to queue one more unacknowledged Accept. - let storm = UdpSocket::bind("127.0.0.1:0")?; - let mut storm_client = cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { - connection_id: 102, - initial_sequence: 1, - resend_delay_ms: 10, - max_pending_reliable_packets: None, - }); - let connect_packet = storm_client.create_connect(0, Vec::new())?; + // each one asks the session to queue one more unacknowledged Accept, and + // the fifth finds the queue full. + let mut stormer = session(102); + let connect_packet = stormer.create_connect(0, Vec::new())?; for _ in 0..12 { - storm.send_to(&cultnet_rs::encode_rudp_packet(&connect_packet)?, target)?; - server.poll_once()?; + send_raw(&mut server, &connect_packet)?; } - storm.send_to(b"not a rudp packet", target)?; - server.poll_once()?; + assert_eq!(server.packets_dropped(), 2 + 8); + assert_eq!(server.session_count(), 2); + + // A packet its own admitted session refuses ends that session, not the loop. + stormer.assume_connected(0); + let mut poison = stormer.send("schema", vec![1], reliable)?; + poison.fragment_count = 2; + poison.fragment_id = 0; + send_raw(&mut server, &poison)?; + assert_eq!(server.packets_dropped(), 2 + 8 + 1); + assert_eq!(server.session_count(), 1); send( &mut real, @@ -598,6 +636,5 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { } } assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); - assert!(server.packets_dropped() > 0, "the storm and the stray are counted"); Ok(()) } From 5fb3fca2baf7ff256fd3df53909069a5699361a9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:14:24 +0200 Subject: [PATCH 27/64] Normalise a blank name once, where it is read The held-ancestor path compared the raw empty string, so a blank variant name and a blank held base name never met the blank rule; NameRead now returns null for blank on both paths. Co-Authored-By: Claude Sonnet 5 --- src/GameCult.Caching/CultCache.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index f9497cb3..245e9367 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2549,9 +2549,7 @@ namespace GameCult.Caching foreach (var stored in plan.Admitted.Concat(plan.Dependents)) { var descriptor = stored.Descriptor; - string? name = null; - if (descriptor.NameAccessor != null) - name = NameRead(stored) is { Length: > 0 } read ? read : null; + var name = descriptor.NameAccessor == null ? null : NameRead(stored); var indexes = new List<(string Alias, string Value)>(); foreach (var pair in descriptor.IndexAccessors) { @@ -2565,8 +2563,9 @@ namespace GameCult.Caching } } + // A blank name is no name. private static string? NameRead(CultStoredDocument stored) => - Read(stored, $"[CultName] member {stored.Descriptor.NameMember}", stored.Descriptor.NameAccessor!); + Read(stored, $"[CultName] member {stored.Descriptor.NameMember}", stored.Descriptor.NameAccessor!) is { Length: > 0 } name ? name : null; private static string? Read(CultStoredDocument stored, string what, Func getter) { From 2ef7ae92c4cff5bc569f983fff4e7ff9c5077aee Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:21:58 +0200 Subject: [PATCH 28/64] cultnet-rs: pin every RUDP stray drop site with an exact count Co-Authored-By: Claude Sonnet 5 --- .../cultnet-rs/tests/rudp_stray_packets.rs | 133 +++++++++++++++++- 1 file changed, 130 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 58f491fa..5dc39732 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -50,6 +50,24 @@ fn send_to(socket: &UdpSocket, to: SocketAddr, packet: &CultNetRudpPacket) -> Re Ok(()) } +/// How many datagrams `send_stray` puts on the wire for `kind`. +fn datagrams(kind: Stray) -> u64 { + match kind { + Stray::Malformed => 2, + Stray::ForeignConnectionId => 4, + Stray::UnadmittedSender => 3, + } +} + +/// A packet the sender's own session accepted for sending and its receiver's +/// session refuses: a fragment with no fragment id. +fn poisoned(session: &mut CultNetRudpSession) -> Result { + let mut packet = session.send("schema", b"poison".to_vec(), reliable())?; + packet.fragment_count = 2; + packet.fragment_id = 0; + Ok(packet) +} + fn send_stray(kind: Stray, from: &UdpSocket, to: SocketAddr) -> Result<()> { match kind { Stray::Malformed => { @@ -98,7 +116,7 @@ fn hub_keeps_serving_its_session(kind: Stray) -> Result<()> { send_stray(kind, &stray, hub_addr)?; std::thread::sleep(Duration::from_millis(20)); while hub.receive_event_once()?.is_some() {} - assert!(hub.stats().packets_dropped > 0, "strays are counted"); + assert_eq!(hub.stats().packets_dropped, datagrams(kind), "every stray is counted once"); real.send("schema", b"still here".to_vec())?; std::thread::sleep(Duration::from_millis(20)); @@ -174,7 +192,7 @@ fn client_keeps_its_session(kind: Stray) -> Result<()> { .receive_once()? .expect("the real frame behind the strays"); assert_eq!(frame.payload, b"real"); - assert!(client.stats().packets_dropped > 0, "strays are counted"); + assert_eq!(client.stats().packets_dropped, datagrams(kind), "every stray is counted once"); Ok(()) } @@ -206,7 +224,8 @@ fn server_mode_keeps_its_peer(kind: Stray) -> Result<()> { .receive_once()? .expect("the real frame behind the strays"); assert_eq!(frame.payload, b"real"); - assert!(server.stats().packets_dropped > 0, "strays are counted"); + // Once from an unrelated address, once from the peer's own. + assert_eq!(server.stats().packets_dropped, 2 * datagrams(kind), "every stray is counted once"); Ok(()) } @@ -253,3 +272,111 @@ per_stray!( server_mode_survives_malformed_frames, server_mode_survives_foreign_connection_ids ); + +#[test] +fn hub_drops_a_packet_its_admitted_peer_session_refuses() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( + "hub", + socket()?, + CONNECTION_ID, + ))?; + let hub_addr = hub.local_addr()?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + send_to(&peer, hub_addr, &peer_session.create_connect(0, b"peer".to_vec())?)?; + assert!(matches!( + hub.receive_event_once()?, + Some(CultNetRudpServerEvent::Connected { .. }) + )); + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + peer_session.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + + send_to(&peer, hub_addr, &poisoned(&mut peer_session)?)?; + assert!(hub.receive_event_once()?.is_none()); + assert_eq!(hub.stats().packets_dropped, 1); + assert_eq!(hub.sessions().len(), 1); + + send_to( + &peer, + hub_addr, + &peer_session.send("schema", b"still here".to_vec(), reliable())?, + )?; + match hub.receive_event_once()? { + Some(CultNetRudpServerEvent::Frame { frame, .. }) => { + assert_eq!(frame.payload, b"still here"); + } + other => panic!("the peer must still be served, got {other:?}"), + } + Ok(()) +} + +#[test] +fn client_drops_a_packet_its_session_refuses() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server.recv_from(&mut buffer)?; + let mut server_session = raw_session(CONNECTION_ID); + let accept = + server_session.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + send_to(&server, client_addr, &poisoned(&mut server_session)?)?; + assert!(client.receive_once()?.is_none()); + assert_eq!(client.stats().packets_dropped, 1); + send_to( + &server, + client_addr, + &server_session.send("schema", b"real".to_vec(), reliable())?, + )?; + assert_eq!(client.receive_once()?.expect("the real frame").payload, b"real"); + Ok(()) +} + +#[test] +fn server_mode_drops_what_is_not_its_peer_or_what_its_session_refuses() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let stray = socket()?; + + // Before any peer: a right-id packet that is not a Connect claims nothing. + send_stray(Stray::UnadmittedSender, &stray, server_addr)?; + assert!(server.receive_once()?.is_none()); + assert_eq!(server.stats().packets_dropped, 3); + + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &peer_session.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + peer_session.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + assert_eq!(server.stats().packets_dropped, 3); + + // After the peer is known: right-id packets from any other address. + send_stray(Stray::UnadmittedSender, &stray, server_addr)?; + assert!(server.receive_once()?.is_none()); + assert_eq!(server.stats().packets_dropped, 6); + + // A packet from the peer that the session itself refuses. + send_to(&peer, server_addr, &poisoned(&mut peer_session)?)?; + assert!(server.receive_once()?.is_none()); + assert_eq!(server.stats().packets_dropped, 7); + + send_to( + &peer, + server_addr, + &peer_session.send("schema", b"real".to_vec(), reliable())?, + )?; + assert_eq!(server.receive_once()?.expect("the real frame").payload, b"real"); + Ok(()) +} From fee55e89c55a377f38248aabd09654b17d85bb87 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:31:16 +0200 Subject: [PATCH 29/64] Check the CultMath release gate against the type-definition table scripts/check-cultmath-types.cs reads the assembly with System.Reflection.Metadata and reports which named types are not public top-level CultMath types. The old NUL-delimited string search passed an appended string or a member named like the type, and failed a real type whose name is tail-shared in the string heap. Prerelease and range versions now get a clear message instead of a raw System.Version error. Co-Authored-By: Claude Sonnet 5 --- scripts/build-unity-package.ps1 | 22 ++++++++++++++-------- scripts/check-cultmath-types.cs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 scripts/check-cultmath-types.cs diff --git a/scripts/build-unity-package.ps1 b/scripts/build-unity-package.ps1 index 49bf391f..9de7c718 100644 --- a/scripts/build-unity-package.ps1 +++ b/scripts/build-unity-package.ps1 @@ -58,15 +58,20 @@ if ($LASTEXITCODE -ne 0) { # dependency, so a cultlib release must not run ahead of the CultMath package it names. # (1) The declared dependency version must not exceed the CultMath package's own version. # (2) The CultMath.dll that package tracks must define every type CultMathResolver formats. -# Type names come from the resolver source (its Shape entries) and are looked up as -# NUL-delimited names in the assembly's metadata string heap, which is a name check, not a -# signature check; Windows PowerShell cannot load a netstandard2.1 assembly to reflect on it. +# The names come from the resolver source (its Shape entries); scripts\check-cultmath-types.cs +# looks them up in the assembly's type-definition table, which Windows PowerShell cannot load +# the assembly to do. It checks type names, not signatures. $cultMathPackageRoot = Join-Path $repoRoot "packages\cultmath\unity\org.gamecult.cultmath" $cultMathDeclared = (Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json).dependencies.'org.gamecult.cultmath' $cultMathAvailable = (Get-Content -LiteralPath (Join-Path $cultMathPackageRoot "package.json") -Raw | ConvertFrom-Json).version if ([string]::IsNullOrWhiteSpace($cultMathDeclared)) { throw "package.json must declare org.gamecult.cultmath, which supplies CultMath.dll." } +foreach ($declared in @(@("org.gamecult.cultlib's org.gamecult.cultmath dependency", $cultMathDeclared), @("the CultMath package version", $cultMathAvailable))) { + if ($declared[1] -notmatch '^\d+\.\d+\.\d+$') { + throw "$($declared[0]) is '$($declared[1])'. The release gate compares exact release versions (major.minor.patch); prerelease and range versions are not supported." + } +} if ([version]$cultMathDeclared -gt [version]$cultMathAvailable) { throw "Release order: org.gamecult.cultlib depends on org.gamecult.cultmath $cultMathDeclared, but the CultMath package is at $cultMathAvailable. Release CultMath first." } @@ -75,11 +80,12 @@ $resolverTypes = @([regex]::Matches($resolverSource, '\bShape<(?!T>)(\w+)>\(') | if ($resolverTypes.Count -eq 0) { throw "Found no CultMath types in CultMathResolver.cs; the release gate cannot check the CultMath package." } -$cultMathBytes = [IO.File]::ReadAllBytes((Join-Path $cultMathPackageRoot "Runtime\Plugins\CultMath.dll")) -$cultMathText = [Text.Encoding]::GetEncoding(28591).GetString($cultMathBytes) -$missingTypes = @($resolverTypes | Where-Object { -not $cultMathText.Contains("`0$_`0") }) -if ($missingTypes.Count -gt 0) { - throw "The CultMath.dll in org.gamecult.cultmath $cultMathAvailable lacks types CultMathResolver formats: $($missingTypes -join ', '). Release CultMath first." +$missingTypes = & dotnet run (Join-Path $PSScriptRoot "check-cultmath-types.cs") -- (Join-Path $cultMathPackageRoot "Runtime\Plugins\CultMath.dll") @resolverTypes +if ($LASTEXITCODE -eq 1) { + throw "The CultMath.dll in org.gamecult.cultmath $cultMathAvailable lacks types CultMathResolver formats: $missingTypes. Release CultMath first." +} +if ($LASTEXITCODE -ne 0) { + throw "scripts\check-cultmath-types.cs failed with exit code $LASTEXITCODE." } # The tracked DLLs and pdbs are committed beside their source, so none may name a commit or a worktree: # Source Link writes the commit SHA into the pdb, the informational version carries it too, and each DLL diff --git a/scripts/check-cultmath-types.cs b/scripts/check-cultmath-types.cs new file mode 100644 index 00000000..dab33397 --- /dev/null +++ b/scripts/check-cultmath-types.cs @@ -0,0 +1,30 @@ +// Release gate helper for build-unity-package.ps1: reads an assembly's type-definition table and +// reports which of the named CultMath types it does not define as a public top-level type. +// Usage: dotnet run scripts/check-cultmath-types.cs -- ... +// Exit 0 when every type is defined, 1 when any is missing, 2 on bad usage. +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +if (args.Length < 2) +{ + Console.Error.WriteLine("usage: check-cultmath-types ..."); + return 2; +} + +using var stream = File.OpenRead(args[0]); +using var pe = new PEReader(stream); +var metadata = pe.GetMetadataReader(); +var defined = new HashSet(); +foreach (var handle in metadata.TypeDefinitions) +{ + var type = metadata.GetTypeDefinition(handle); + if (!type.GetDeclaringType().IsNil) continue; + if ((type.Attributes & TypeAttributes.VisibilityMask) != TypeAttributes.Public) continue; + if (metadata.GetString(type.Namespace) == "CultMath") defined.Add(metadata.GetString(type.Name)); +} + +var missing = args.Skip(1).Where(name => !defined.Contains(name)).ToArray(); +if (missing.Length == 0) return 0; +Console.WriteLine(string.Join(", ", missing)); +return 1; From 4e5327c61c43779db7bfe08b8bd404b451a84fa7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:31:16 +0200 Subject: [PATCH 30/64] Bring the CultMath JSON converters to System.Text.Json parity and remove their allocations Named literals now follow AllowReadingFromString and WriteAsString as System.Text.Json's float does; quoted numbers refuse whitespace; UnmappedMemberHandling.Disallow and AllowDuplicateProperties=false are honoured and a repeated property replaces the whole value; dictionary keys read only the canonical spelling. Components live in stack spans, property names are pre-encoded, and there is no per-property closure, so a float3 read allocates nothing (424 bytes before). Tests pin unknown nested values, de-DE and sv-SE style cultures, and a distinct negative-zero key. Co-Authored-By: Claude Sonnet 5 --- .../CultMathJson.cs | 474 +++++++++--------- src/GameCult.Caching.MessagePack/README.md | 24 +- .../CultMathJsonTests.cs | 185 +++++++ 3 files changed, 449 insertions(+), 234 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultMathJson.cs b/src/GameCult.Caching.MessagePack/CultMathJson.cs index def7ab9a..438c6150 100644 --- a/src/GameCult.Caching.MessagePack/CultMathJson.cs +++ b/src/GameCult.Caching.MessagePack/CultMathJson.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Reflection; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using CultMath; @@ -16,22 +18,30 @@ namespace GameCult.Caching.MessagePack; /// Property names are fixed: PropertyNamingPolicy does not rename them, and /// PropertyNameCaseInsensitive makes reading case-insensitive. /// -/// Reading skips unknown properties and leaves missing ones at zero (a missing Color32.a -/// is 255). A wrong token, a number outside its component type (including a float beyond -/// , which is refused rather than becoming infinity), or a matrix -/// that is not an array of exactly its row count throws . +/// Reading skips unknown properties (UnmappedMemberHandling.Disallow refuses them) and +/// leaves missing ones at zero (a missing Color32.a is 255). A repeated property replaces +/// the whole earlier value, never merging component-wise, and is refused when +/// AllowDuplicateProperties is false. A wrong token, a number outside its component type, +/// or a matrix that is not an array of exactly its row count throws +/// . /// /// /// Numbers follow as System.Text.Json's own -/// float does: NaN and the infinities need AllowNamedFloatingPointLiterals (written and -/// read as "NaN", "Infinity", "-Infinity") and are otherwise refused with -/// ; AllowReadingFromString reads quoted numbers; -/// WriteAsString writes quoted numbers. +/// float does. NaN and the infinities are the strings "NaN", "Infinity", +/// "-Infinity": written when AllowNamedFloatingPointLiterals or +/// WriteAsString is set (otherwise ), read when +/// AllowNamedFloatingPointLiterals or AllowReadingFromString is set. +/// AllowReadingFromString reads quoted numbers without surrounding whitespace, and +/// WriteAsString writes quoted numbers. Two intended divergences: a bare number beyond +/// the component's range (1e40 as a float) is refused where System.Text.Json reads +/// infinity, and refusals on write are where it throws +/// . /// /// /// Every type is also a dictionary key: the flattened components joined by commas in /// declaration order, invariant culture, shortest round-trip floats ("1,2", /// "true,false", matrices row by row, rect as "minX,minY,maxX,maxY"). +/// Only that exact spelling reads back, so two JSON keys never collapse into one dictionary key. /// /// Register with ; JSON options are the consumer's own. /// @@ -48,150 +58,141 @@ public static class CultMathJson return options; } - private static bool Named(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.AllowNamedFloatingPointLiterals) != 0; + private static bool LiteralsWrite(JsonSerializerOptions o) => + (o.NumberHandling & (JsonNumberHandling.AllowNamedFloatingPointLiterals | JsonNumberHandling.WriteAsString)) != 0; + + private static bool LiteralsRead(JsonSerializerOptions o) => + (o.NumberHandling & (JsonNumberHandling.AllowNamedFloatingPointLiterals | JsonNumberHandling.AllowReadingFromString)) != 0; + private static bool FromString(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.AllowReadingFromString) != 0; private static bool AsString(JsonSerializerOptions o) => (o.NumberHandling & JsonNumberHandling.WriteAsString) != 0; - private static string Invalid(string what, string got) => $"{what} cannot be read from '{got}'."; + // AllowDuplicateProperties arrived after the System.Text.Json this library compiles against. + private static readonly PropertyInfo? AllowDuplicateProperties = typeof(JsonSerializerOptions).GetProperty("AllowDuplicateProperties"); + + private static bool DuplicatesAllowed(JsonSerializerOptions o) => AllowDuplicateProperties == null || (bool)AllowDuplicateProperties.GetValue(o)!; + + private const NumberStyles FloatStyle = NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent; // One component kind. Every CultMath type is a flat list of one kind, so a Scalar plus a Node tree describes it. - private abstract class Scalar + private abstract class Scalar where S : unmanaged { - public virtual S[] Defaults(int count) => new S[count]; public abstract void Write(Utf8JsonWriter w, S value, JsonSerializerOptions o); public abstract S Read(ref Utf8JsonReader r, JsonSerializerOptions o); - public abstract string Text(S value, JsonSerializerOptions o); + + /// The canonical key spelling, whatever the options allow. + public abstract string Spelling(S value); + + /// The key spelling, refusing what the options would refuse to write. + public virtual string Text(S value, JsonSerializerOptions o) => Spelling(value); + public abstract S Parse(string text, JsonSerializerOptions o); + + /// Reads a key component; keys are always text, so AllowReadingFromString does not apply. + public virtual S ParseKey(string text, JsonSerializerOptions o) => Parse(text, o); } - private sealed class FloatScalar : Scalar + private abstract class FloatingScalar : Scalar where S : unmanaged + { + protected static bool IsLiteral(string s) => s == "NaN" || s == "Infinity" || s == "-Infinity"; + protected abstract bool IsFinite(S value); + protected abstract string Literal(S value); + protected abstract S FromLiteral(string literal); + protected abstract bool TryNumber(string text, out S value); + protected abstract bool TryToken(ref Utf8JsonReader r, out S value); + protected abstract void WriteNumber(Utf8JsonWriter w, S value); + protected abstract string Number(S value); + + public override void Write(Utf8JsonWriter w, S v, JsonSerializerOptions o) + { + if (!IsFinite(v)) + { + if (!LiteralsWrite(o)) throw new JsonException($"A non-finite {typeof(S).Name} needs JsonNumberHandling.AllowNamedFloatingPointLiterals or WriteAsString."); + w.WriteStringValue(Literal(v)); + } + else if (AsString(o)) w.WriteStringValue(Number(v)); + else WriteNumber(w, v); + } + + public override S Read(ref Utf8JsonReader r, JsonSerializerOptions o) + { + if (r.TokenType == JsonTokenType.Number) + { + if (!TryToken(ref r, out var v) || !IsFinite(v)) throw new JsonException($"The number is outside the {typeof(S).Name} range."); + return v; + } + + if (r.TokenType == JsonTokenType.String) return Parse(r.GetString()!, o); + throw new JsonException($"Expected a number, found {r.TokenType}."); + } + + public override string Spelling(S v) => Number(v); + + public override string Text(S v, JsonSerializerOptions o) + { + if (!IsFinite(v) && !LiteralsWrite(o)) throw new JsonException($"A non-finite {typeof(S).Name} needs JsonNumberHandling.AllowNamedFloatingPointLiterals or WriteAsString."); + return Spelling(v); + } + + public override S Parse(string text, JsonSerializerOptions o) + { + if (!IsLiteral(text) && !FromString(o)) throw new JsonException($"Expected a number, found the string '{text}'."); + return ParseKey(text, o); + } + + public override S ParseKey(string text, JsonSerializerOptions o) + { + if (IsLiteral(text)) + { + if (!LiteralsRead(o)) throw new JsonException($"'{text}' needs JsonNumberHandling.AllowNamedFloatingPointLiterals or AllowReadingFromString."); + return FromLiteral(text); + } + + if (TryNumber(text, out var v) && IsFinite(v)) return v; + throw new JsonException($"A {typeof(S).Name} cannot be read from '{text}'."); + } + } + + private sealed class FloatScalar : FloatingScalar { public static readonly FloatScalar Instance = new(); - - private static bool IsLiteral(string s) => s == "NaN" || s == "Infinity" || s == "-Infinity"; - - private static string Literal(float v) => float.IsNaN(v) ? "NaN" : v > 0f ? "Infinity" : "-Infinity"; - - public override void Write(Utf8JsonWriter w, float v, JsonSerializerOptions o) - { - if (!float.IsFinite(v)) - { - if (!Named(o)) throw new JsonException("A non-finite float needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); - w.WriteStringValue(Literal(v)); - } - else if (AsString(o)) w.WriteStringValue(v.ToString("R", CultureInfo.InvariantCulture)); - else w.WriteNumberValue(v); - } - - public override float Read(ref Utf8JsonReader r, JsonSerializerOptions o) - { - if (r.TokenType == JsonTokenType.Number) - { - if (!r.TryGetSingle(out var v) || float.IsInfinity(v)) throw new JsonException("The number is outside the float range."); - return v; - } - - if (r.TokenType == JsonTokenType.String) - { - var text = r.GetString()!; - if (!FromString(o) && !(Named(o) && IsLiteral(text))) throw new JsonException($"Expected a number, found the string '{text}'."); - return Parse(text, o); - } - - throw new JsonException($"Expected a number, found {r.TokenType}."); - } - - public override string Text(float v, JsonSerializerOptions o) - { - if (!float.IsFinite(v) && !Named(o)) throw new JsonException("A non-finite float needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); - return v.ToString("R", CultureInfo.InvariantCulture); - } - - public override float Parse(string text, JsonSerializerOptions o) - { - if (Named(o)) - { - if (text == "NaN") return float.NaN; - if (text == "Infinity") return float.PositiveInfinity; - if (text == "-Infinity") return float.NegativeInfinity; - } - - if (float.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var v) && float.IsFinite(v)) return v; - throw new JsonException(Invalid("A float", text)); - } + protected override bool IsFinite(float v) => float.IsFinite(v); + protected override string Literal(float v) => float.IsNaN(v) ? "NaN" : v > 0f ? "Infinity" : "-Infinity"; + protected override float FromLiteral(string s) => s == "NaN" ? float.NaN : s == "Infinity" ? float.PositiveInfinity : float.NegativeInfinity; + protected override bool TryNumber(string s, out float v) => float.TryParse(s, FloatStyle, CultureInfo.InvariantCulture, out v); + protected override bool TryToken(ref Utf8JsonReader r, out float v) => r.TryGetSingle(out v); + protected override void WriteNumber(Utf8JsonWriter w, float v) => w.WriteNumberValue(v); + protected override string Number(float v) => v.ToString("R", CultureInfo.InvariantCulture); } - private sealed class DoubleScalar : Scalar + private sealed class DoubleScalar : FloatingScalar { public static readonly DoubleScalar Instance = new(); - - private static bool IsLiteral(string s) => s == "NaN" || s == "Infinity" || s == "-Infinity"; - - private static string Literal(double v) => double.IsNaN(v) ? "NaN" : v > 0d ? "Infinity" : "-Infinity"; - - public override void Write(Utf8JsonWriter w, double v, JsonSerializerOptions o) - { - if (!double.IsFinite(v)) - { - if (!Named(o)) throw new JsonException("A non-finite double needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); - w.WriteStringValue(Literal(v)); - } - else if (AsString(o)) w.WriteStringValue(v.ToString("R", CultureInfo.InvariantCulture)); - else w.WriteNumberValue(v); - } - - public override double Read(ref Utf8JsonReader r, JsonSerializerOptions o) - { - if (r.TokenType == JsonTokenType.Number) - { - if (!r.TryGetDouble(out var v) || double.IsInfinity(v)) throw new JsonException("The number is outside the double range."); - return v; - } - - if (r.TokenType == JsonTokenType.String) - { - var text = r.GetString()!; - if (!FromString(o) && !(Named(o) && IsLiteral(text))) throw new JsonException($"Expected a number, found the string '{text}'."); - return Parse(text, o); - } - - throw new JsonException($"Expected a number, found {r.TokenType}."); - } - - public override string Text(double v, JsonSerializerOptions o) - { - if (!double.IsFinite(v) && !Named(o)) throw new JsonException("A non-finite double needs JsonNumberHandling.AllowNamedFloatingPointLiterals."); - return v.ToString("R", CultureInfo.InvariantCulture); - } - - public override double Parse(string text, JsonSerializerOptions o) - { - if (Named(o)) - { - if (text == "NaN") return double.NaN; - if (text == "Infinity") return double.PositiveInfinity; - if (text == "-Infinity") return double.NegativeInfinity; - } - - if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var v) && double.IsFinite(v)) return v; - throw new JsonException(Invalid("A double", text)); - } + protected override bool IsFinite(double v) => double.IsFinite(v); + protected override string Literal(double v) => double.IsNaN(v) ? "NaN" : v > 0d ? "Infinity" : "-Infinity"; + protected override double FromLiteral(string s) => s == "NaN" ? double.NaN : s == "Infinity" ? double.PositiveInfinity : double.NegativeInfinity; + protected override bool TryNumber(string s, out double v) => double.TryParse(s, FloatStyle, CultureInfo.InvariantCulture, out v); + protected override bool TryToken(ref Utf8JsonReader r, out double v) => r.TryGetDouble(out v); + protected override void WriteNumber(Utf8JsonWriter w, double v) => w.WriteNumberValue(v); + protected override string Number(double v) => v.ToString("R", CultureInfo.InvariantCulture); } - private sealed class IntScalar : Scalar + private abstract class IntegerScalar : Scalar where S : unmanaged { - public static readonly IntScalar Instance = new(); + protected abstract bool TryToken(ref Utf8JsonReader r, out S value); + protected abstract bool TryText(string text, out S value); + protected abstract void WriteNumber(Utf8JsonWriter w, S value); - public override void Write(Utf8JsonWriter w, int v, JsonSerializerOptions o) + public override void Write(Utf8JsonWriter w, S v, JsonSerializerOptions o) { - if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); + if (AsString(o)) w.WriteStringValue(Spelling(v)); else WriteNumber(w, v); } - public override int Read(ref Utf8JsonReader r, JsonSerializerOptions o) + public override S Read(ref Utf8JsonReader r, JsonSerializerOptions o) { if (r.TokenType == JsonTokenType.Number) { - if (!r.TryGetInt32(out var v)) throw new JsonException("The number is not an int."); + if (!TryToken(ref r, out var v)) throw new JsonException($"The number is not a {typeof(S).Name}."); return v; } @@ -199,71 +200,35 @@ public static class CultMathJson throw new JsonException($"Expected an integer, found {r.TokenType}."); } - public override string Text(int v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); - - public override int Parse(string text, JsonSerializerOptions o) => - int.TryParse(text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("An int", text)); + public override S Parse(string text, JsonSerializerOptions o) => + TryText(text, out var v) ? v : throw new JsonException($"A {typeof(S).Name} cannot be read from '{text}'."); } - private sealed class UIntScalar : Scalar + private sealed class IntScalar : IntegerScalar + { + public static readonly IntScalar Instance = new(); + protected override bool TryToken(ref Utf8JsonReader r, out int v) => r.TryGetInt32(out v); + protected override bool TryText(string s, out int v) => int.TryParse(s, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out v); + protected override void WriteNumber(Utf8JsonWriter w, int v) => w.WriteNumberValue(v); + public override string Spelling(int v) => v.ToString(CultureInfo.InvariantCulture); + } + + private sealed class UIntScalar : IntegerScalar { public static readonly UIntScalar Instance = new(); - - public override void Write(Utf8JsonWriter w, uint v, JsonSerializerOptions o) - { - if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); - } - - public override uint Read(ref Utf8JsonReader r, JsonSerializerOptions o) - { - if (r.TokenType == JsonTokenType.Number) - { - if (!r.TryGetUInt32(out var v)) throw new JsonException("The number is not a uint."); - return v; - } - - if (r.TokenType == JsonTokenType.String && FromString(o)) return Parse(r.GetString()!, o); - throw new JsonException($"Expected an unsigned integer, found {r.TokenType}."); - } - - public override string Text(uint v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); - - public override uint Parse(string text, JsonSerializerOptions o) => - uint.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("A uint", text)); + protected override bool TryToken(ref Utf8JsonReader r, out uint v) => r.TryGetUInt32(out v); + protected override bool TryText(string s, out uint v) => uint.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out v); + protected override void WriteNumber(Utf8JsonWriter w, uint v) => w.WriteNumberValue(v); + public override string Spelling(uint v) => v.ToString(CultureInfo.InvariantCulture); } - private sealed class ByteScalar : Scalar + private sealed class ByteScalar : IntegerScalar { public static readonly ByteScalar Instance = new(); - - public override byte[] Defaults(int count) - { - var defaults = new byte[count]; - if (count == 4) defaults[3] = 255; // Color32 alpha - return defaults; - } - - public override void Write(Utf8JsonWriter w, byte v, JsonSerializerOptions o) - { - if (AsString(o)) w.WriteStringValue(v.ToString(CultureInfo.InvariantCulture)); else w.WriteNumberValue(v); - } - - public override byte Read(ref Utf8JsonReader r, JsonSerializerOptions o) - { - if (r.TokenType == JsonTokenType.Number) - { - if (!r.TryGetByte(out var v)) throw new JsonException("The number is not a byte."); - return v; - } - - if (r.TokenType == JsonTokenType.String && FromString(o)) return Parse(r.GetString()!, o); - throw new JsonException($"Expected a byte, found {r.TokenType}."); - } - - public override string Text(byte v, JsonSerializerOptions o) => v.ToString(CultureInfo.InvariantCulture); - - public override byte Parse(string text, JsonSerializerOptions o) => - byte.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var v) ? v : throw new JsonException(Invalid("A byte", text)); + protected override bool TryToken(ref Utf8JsonReader r, out byte v) => r.TryGetByte(out v); + protected override bool TryText(string s, out byte v) => byte.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out v); + protected override void WriteNumber(Utf8JsonWriter w, byte v) => w.WriteNumberValue(v); + public override string Spelling(byte v) => v.ToString(CultureInfo.InvariantCulture); } private sealed class BoolScalar : Scalar @@ -277,76 +242,115 @@ public static class CultMathJson : r.TokenType == JsonTokenType.False ? false : throw new JsonException($"Expected true or false, found {r.TokenType}."); - public override string Text(bool v, JsonSerializerOptions o) => v ? "true" : "false"; + public override string Spelling(bool v) => v ? "true" : "false"; public override bool Parse(string text, JsonSerializerOptions o) => - text == "true" ? true : text == "false" ? false : throw new JsonException(Invalid("A bool", text)); + text == "true" ? true : text == "false" ? false : throw new JsonException($"A bool cannot be read from '{text}'."); } // The JSON structure over a flat component list: leaves consume one component each, in order. - private abstract class Node + // Read fills `flat` (pre-loaded with `defaults`), so a missing property keeps its default. + private abstract class Node where S : unmanaged { public int Count { get; protected set; } - public abstract void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o); - public abstract void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o); + public abstract void Write(Utf8JsonWriter w, ReadOnlySpan flat, Scalar scalar, JsonSerializerOptions o); + public abstract void Read(ref Utf8JsonReader r, scoped Span flat, scoped ReadOnlySpan defaults, Scalar scalar, JsonSerializerOptions o); } - private sealed class Leaf : Node + private sealed class Leaf : Node where S : unmanaged { public Leaf() { Count = 1; } - public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) => scalar.Write(w, flat[offset], o); + public override void Write(Utf8JsonWriter w, ReadOnlySpan flat, Scalar scalar, JsonSerializerOptions o) => scalar.Write(w, flat[0], o); - public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) => flat[offset] = scalar.Read(ref r, o); + public override void Read(ref Utf8JsonReader r, scoped Span flat, scoped ReadOnlySpan defaults, Scalar scalar, JsonSerializerOptions o) => flat[0] = scalar.Read(ref r, o); } - private sealed class Fields : Node + private sealed class Fields : Node where S : unmanaged { private readonly string[] names; + private readonly byte[][] utf8Names; + private readonly JsonEncodedText[] encoded; private readonly Node[] children; private readonly int[] offsets; public Fields(params (string Name, Node Child)[] fields) { names = new string[fields.Length]; + utf8Names = new byte[fields.Length][]; + encoded = new JsonEncodedText[fields.Length]; children = new Node[fields.Length]; offsets = new int[fields.Length]; for (var i = 0; i < fields.Length; i++) { names[i] = fields[i].Name; + utf8Names[i] = Encoding.UTF8.GetBytes(fields[i].Name); + encoded[i] = JsonEncodedText.Encode(fields[i].Name); children[i] = fields[i].Child; offsets[i] = Count; Count += fields[i].Child.Count; } } - public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + public override void Write(Utf8JsonWriter w, ReadOnlySpan flat, Scalar scalar, JsonSerializerOptions o) { w.WriteStartObject(); - for (var i = 0; i < names.Length; i++) + for (var i = 0; i < children.Length; i++) { - w.WritePropertyName(names[i]); - children[i].Write(w, flat, offset + offsets[i], scalar, o); + w.WritePropertyName(encoded[i]); + children[i].Write(w, flat.Slice(offsets[i], children[i].Count), scalar, o); } w.WriteEndObject(); } - public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + public override void Read(ref Utf8JsonReader r, scoped Span flat, scoped ReadOnlySpan defaults, Scalar scalar, JsonSerializerOptions o) { if (r.TokenType != JsonTokenType.StartObject) throw new JsonException($"Expected a JSON object, found {r.TokenType}."); - var comparison = o.PropertyNameCaseInsensitive ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var ignoreCase = o.PropertyNameCaseInsensitive; + var seen = 0; while (r.Read() && r.TokenType == JsonTokenType.PropertyName) { - var name = r.GetString()!; - var index = Array.FindIndex(names, n => string.Equals(n, name, comparison)); + var index = IndexOf(ref r, ignoreCase); r.Read(); - if (index < 0) r.Skip(); else children[index].Read(ref r, flat, offset + offsets[index], scalar, o); + if (index < 0) + { + if (o.UnmappedMemberHandling == JsonUnmappedMemberHandling.Disallow) + throw new JsonException("The JSON object has a property that is not a component."); + r.Skip(); + continue; + } + + var slice = flat.Slice(offsets[index], children[index].Count); + var bit = 1 << index; + if ((seen & bit) != 0) + { + if (!DuplicatesAllowed(o)) throw new JsonException($"The property '{names[index]}' appears more than once."); + defaults.Slice(offsets[index], slice.Length).CopyTo(slice); + } + + seen |= bit; + children[index].Read(ref r, slice, defaults.Slice(offsets[index], slice.Length), scalar, o); } } + + private int IndexOf(ref Utf8JsonReader r, bool ignoreCase) + { + if (!ignoreCase) + { + for (var i = 0; i < utf8Names.Length; i++) + if (r.ValueTextEquals(utf8Names[i])) return i; + return -1; + } + + var name = r.GetString()!; + for (var i = 0; i < names.Length; i++) + if (string.Equals(names[i], name, StringComparison.OrdinalIgnoreCase)) return i; + return -1; + } } - private sealed class Rows : Node + private sealed class Rows : Node where S : unmanaged { private readonly Node row; private readonly int rows; @@ -358,81 +362,98 @@ public static class CultMathJson Count = rows * row.Count; } - public override void Write(Utf8JsonWriter w, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + public override void Write(Utf8JsonWriter w, ReadOnlySpan flat, Scalar scalar, JsonSerializerOptions o) { w.WriteStartArray(); - for (var i = 0; i < rows; i++) row.Write(w, flat, offset + (i * row.Count), scalar, o); + for (var i = 0; i < rows; i++) row.Write(w, flat.Slice(i * row.Count, row.Count), scalar, o); w.WriteEndArray(); } - public override void Read(ref Utf8JsonReader r, S[] flat, int offset, Scalar scalar, JsonSerializerOptions o) + public override void Read(ref Utf8JsonReader r, scoped Span flat, scoped ReadOnlySpan defaults, Scalar scalar, JsonSerializerOptions o) { if (r.TokenType != JsonTokenType.StartArray) throw new JsonException($"Expected a JSON array of {rows} rows, found {r.TokenType}."); for (var i = 0; i < rows; i++) { if (!r.Read() || r.TokenType == JsonTokenType.EndArray) throw new JsonException($"Expected {rows} rows."); - row.Read(ref r, flat, offset + (i * row.Count), scalar, o); + row.Read(ref r, flat.Slice(i * row.Count, row.Count), defaults.Slice(i * row.Count, row.Count), scalar, o); } if (!r.Read() || r.TokenType != JsonTokenType.EndArray) throw new JsonException($"Expected {rows} rows."); } } - private sealed class Shape : JsonConverter + private delegate T Make(ReadOnlySpan components) where S : unmanaged; + private delegate void Flatten(T value, Span components) where S : unmanaged; + + private sealed class Shape : JsonConverter where S : unmanaged { private readonly Scalar scalar; private readonly Node root; - private readonly Func make; - private readonly Action flatten; + private readonly S[] defaults; + private readonly Make make; + private readonly Flatten flatten; - public Shape(Scalar scalar, Node root, Func make, Action flatten) + public Shape(Scalar scalar, Node root, Make make, Flatten flatten, S[]? initial) { this.scalar = scalar; this.root = root; this.make = make; this.flatten = flatten; + defaults = initial ?? new S[root.Count]; } public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - var flat = scalar.Defaults(root.Count); - root.Read(ref reader, flat, 0, scalar, options); + Span flat = stackalloc S[root.Count]; + defaults.CopyTo(flat); + root.Read(ref reader, flat, defaults, scalar, options); return make(flat); } public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) { - var flat = new S[root.Count]; + Span flat = stackalloc S[root.Count]; flatten(value, flat); - root.Write(writer, flat, 0, scalar, options); + root.Write(writer, flat, scalar, options); } public override T ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - var parts = (reader.GetString() ?? throw new JsonException("A dictionary key cannot be null.")).Split(','); + var text = reader.GetString() ?? throw new JsonException("A dictionary key cannot be null."); + var parts = text.Split(','); if (parts.Length != root.Count) throw new JsonException($"A {typeof(T).Name} key has {root.Count} comma-separated components."); - var flat = new S[root.Count]; - for (var i = 0; i < flat.Length; i++) flat[i] = scalar.Parse(parts[i], options); - return make(flat); + Span flat = stackalloc S[root.Count]; + for (var i = 0; i < parts.Length; i++) flat[i] = scalar.ParseKey(parts[i], options); + var value = make(flat); + if (Spell(flat) != text) throw new JsonException($"'{text}' is not the canonical spelling of a {typeof(T).Name} key."); + return value; } public override void WriteAsPropertyName(Utf8JsonWriter writer, T value, JsonSerializerOptions options) { - var flat = new S[root.Count]; + Span flat = stackalloc S[root.Count]; flatten(value, flat); - writer.WritePropertyName(string.Join(",", Array.ConvertAll(flat, c => scalar.Text(c, options)))); + for (var i = 0; i < flat.Length; i++) scalar.Text(flat[i], options); + writer.WritePropertyName(Spell(flat)); + } + + private string Spell(ReadOnlySpan flat) + { + var parts = new string[flat.Length]; + for (var i = 0; i < parts.Length; i++) parts[i] = scalar.Spelling(flat[i]); + return string.Join(",", parts); } } - private static Node Val() => new Leaf(); + private static Node Val() where S : unmanaged => new Leaf(); - private static Node Xy() => new Fields(("x", Val()), ("y", Val())); - private static Node Xyz() => new Fields(("x", Val()), ("y", Val()), ("z", Val())); - private static Node Xyzw() => new Fields(("x", Val()), ("y", Val()), ("z", Val()), ("w", Val())); + private static Node Xy() where S : unmanaged => new Fields(("x", Val()), ("y", Val())); + private static Node Xyz() where S : unmanaged => new Fields(("x", Val()), ("y", Val()), ("z", Val())); + private static Node Xyzw() where S : unmanaged => new Fields(("x", Val()), ("y", Val()), ("z", Val()), ("w", Val())); private static KeyValuePair Add( - Scalar scalar, Node root, Func make, Action flatten) => - new(typeof(T), new Shape(scalar, root, make, flatten)); + Scalar scalar, Node root, Make make, Flatten flatten, S[]? initial = null) where S : unmanaged => + new(typeof(T), new Shape(scalar, root, make, flatten, initial)); private static readonly Dictionary Converters = Build(); @@ -455,7 +476,8 @@ public static class CultMathJson Add(BoolScalar.Instance, Xyzw(), a => new bool4(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.x; a[1] = v.y; a[2] = v.z; a[3] = v.w; }), Add(ByteScalar.Instance, new Fields(("r", Val()), ("g", Val()), ("b", Val()), ("a", Val())), - a => new Color32(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.r; a[1] = v.g; a[2] = v.b; a[3] = v.a; }), + a => new Color32(a[0], a[1], a[2], a[3]), (v, a) => { a[0] = v.r; a[1] = v.g; a[2] = v.b; a[3] = v.a; }, + initial: new byte[] { 0, 0, 0, 255 }), Add(UIntScalar.Instance, new Fields(("state", Val())), a => new Random { state = a[0] }, (v, a) => a[0] = v.state), diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index 27e3ea57..fdf91ab2 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -65,14 +65,22 @@ in the default chain) and one JSON shape (`CultMathJson`, added to a consumer's options with `new JsonSerializerOptions().AddCultMathConverters()`; CultLib owns no JSON options of its own). MessagePack decoding skips extra elements, leaves missing ones at zero (`Color32` alpha at 255) and refuses nil. JSON reading -skips unknown properties, leaves missing ones at zero, honours -`PropertyNameCaseInsensitive`, and throws `JsonException` for a wrong token or an -out-of-range number (a float beyond `float.MaxValue` is refused, not read as -infinity). Numbers follow `JsonSerializerOptions.NumberHandling`: NaN and the -infinities need `AllowNamedFloatingPointLiterals` (`"NaN"`, `"Infinity"`, -`"-Infinity"`). Every type is also a dictionary key: its flattened components -joined by commas, invariant culture (`"1,2"`, `"true,false"`, matrices row by -row, `rect` as `"minX,minY,maxX,maxY"`). A consumer resolver or converter +skips unknown properties (`UnmappedMemberHandling.Disallow` refuses them), leaves +missing ones at zero, honours `PropertyNameCaseInsensitive`, lets a repeated +property replace the whole earlier value (refused when `AllowDuplicateProperties` +is false), and throws `JsonException` for a wrong token or an out-of-range number. +Numbers follow `JsonSerializerOptions.NumberHandling` as System.Text.Json's own +float does: `"NaN"`, `"Infinity"` and `"-Infinity"` are written under +`AllowNamedFloatingPointLiterals` or `WriteAsString` and read under +`AllowNamedFloatingPointLiterals` or `AllowReadingFromString`; quoted numbers read +under `AllowReadingFromString` with no surrounding whitespace. +Every type is also a dictionary key: its flattened components joined by commas, +invariant culture (`"1,2"`, `"true,false"`, matrices row by row, `rect` as +`"minX,minY,maxX,maxY"`), and only that exact spelling reads back. Two intended +divergences from System.Text.Json: a bare number beyond a component's range +(`1e40` as a float) is refused where System.Text.Json reads infinity, and a +refusal on write is a `JsonException` where System.Text.Json throws +`ArgumentException`. A consumer resolver or converter registered ahead of these wins. | Type | MessagePack | JSON | diff --git a/tests/GameCult.Caching.Tests/CultMathJsonTests.cs b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs index bf8444f9..87e5b742 100644 --- a/tests/GameCult.Caching.Tests/CultMathJsonTests.cs +++ b/tests/GameCult.Caching.Tests/CultMathJsonTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; @@ -225,6 +226,190 @@ namespace GameCult.Caching.Tests return JsonDocument.Parse(json).RootElement.EnumerateObject().Single().Name; } + private const JsonNumberHandling ReadStrings = JsonNumberHandling.AllowReadingFromString; + private const JsonNumberHandling WriteStrings = JsonNumberHandling.WriteAsString; + + // Every row was measured against System.Text.Json's own float. + [Test] + public void NamedLiterals_FollowTheSameFlagsAsSystemTextJsonFloat() + { + foreach (var handling in new[] { Named, ReadStrings, ReadStrings | Named }) + { + JsonSerializer.Deserialize("""{"x":"NaN","y":"-Infinity"}""", With(handling))!.y.Should().Be(float.NegativeInfinity, handling.ToString()); + } + + foreach (var handling in new[] { JsonNumberHandling.Strict, WriteStrings }) + { + Action read = () => JsonSerializer.Deserialize("""{"x":"NaN"}""", With(handling)); + read.Should().Throw(handling.ToString()); + } + + foreach (var handling in new[] { Named, WriteStrings, WriteStrings | Named }) + { + JsonSerializer.Serialize(new float2(float.NaN, float.PositiveInfinity), With(handling)) + .Should().Be("""{"x":"NaN","y":"Infinity"}""", handling.ToString()); + } + + foreach (var handling in new[] { JsonNumberHandling.Strict, ReadStrings }) + { + Action write = () => JsonSerializer.Serialize(new float2(float.NaN, 0f), With(handling)); + write.Should().Throw(handling.ToString()); + } + + JsonSerializer.Serialize(new float2(1f, float.NaN), With(WriteStrings)).Should().Be("""{"x":"1","y":"NaN"}"""); + JsonSerializer.Deserialize("""{"z":"Infinity"}""", With(ReadStrings))!.z.Should().Be(double.PositiveInfinity); + JsonSerializer.Serialize(new double2(double.NegativeInfinity, 2d), With(WriteStrings)).Should().Be("""{"x":"-Infinity","y":"2"}"""); + } + + [Test] + public void QuotedNumbers_RefuseWhitespace() + { + var options = With(ReadStrings); + foreach (var json in new[] { """{"x":" 1.5"}""", """{"x":"1.5 "}""", """{"x":"\t1"}""" }) + { + Action floatRead = () => JsonSerializer.Deserialize(json, options); + floatRead.Should().Throw(json); + } + + Action doubleRead = () => JsonSerializer.Deserialize("""{"y":" 2"}""", options); + doubleRead.Should().Throw(); + Action intRead = () => JsonSerializer.Deserialize("""{"x":" 1"}""", options); + intRead.Should().Throw(); + Action padded = () => JsonSerializer.Deserialize("""{"x":" NaN"}""", With(ReadStrings | Named)); + padded.Should().Throw(); + } + + [Test] + public void UnknownProperties_AreSkippedWhateverTheyContain() + { + var expected = new float3(1f, 2f, 3f); + foreach (var json in new[] + { + """{"u":{"a":[1,{"b":2}]},"x":1,"y":2,"z":3}""", + """{"x":1,"u":[[1],[2,[3]]],"y":2,"v":{"a":{}},"z":3}""", + """{"x":1,"y":2,"z":3,"u":{"a":{"b":[]}}}""", + """{"x":1,"y":2,"z":3,"u":[{"a":[{}]}]}""", + }) + { + Parse(json).Should().Be(expected, json); + } + + Parse("""{"min":{"q":[{}],"x":1,"y":2},"skip":{"a":[1]},"max":{"x":3,"y":4,"q":{"a":[]}}}""") + .Should().Be(new rect(1f, 2f, 3f, 4f)); + Parse("""[{"x":1,"u":[1],"y":2},{"y":4,"u":{"a":1},"x":3}]""").Should().Be(new float2x2(1f, 2f, 3f, 4f)); + } + + [Test] + public void UnmappedMemberHandling_Disallow_RefusesUnknownProperties() + { + var options = new JsonSerializerOptions { UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow }.AddCultMathConverters(); + Action flat = () => JsonSerializer.Deserialize("""{"x":1,"z":3}""", options); + flat.Should().Throw(); + Action nested = () => JsonSerializer.Deserialize("""{"min":{"x":1,"q":1},"max":{}}""", options); + nested.Should().Throw(); + JsonSerializer.Deserialize("""{"x":1,"y":2}""", options).Should().Be(new float2(1f, 2f)); + } + + [Test] + public void DuplicateProperties_ReplaceTheWholeValue() + { + Parse("""{"x":1,"x":2,"y":5}""").Should().Be(new float2(2f, 5f)); + Parse("""{"min":{"x":1,"y":2},"min":{"x":3},"max":{"x":4,"y":5}}""").Should().Be(new rect(3f, 0f, 4f, 5f)); + Parse("""{"a":9,"a":1}""").Should().Be(new Color32(0, 0, 0, 1)); + Parse("""{"r":3,"r":4}""").Should().Be(new Color32(4, 0, 0, 255)); + Parse("""{"nearest":{"x":1,"y":2},"nearest":{"z":3},"id":1}""") + .Should().Be(new CultCellular(new float4(0f, 0f, 3f, 0f), float4.zero, 1f)); + } + + [Test] + public void AllowDuplicatePropertiesFalse_RefusesRepeats() + { + var property = typeof(JsonSerializerOptions).GetProperty("AllowDuplicateProperties"); + Assert.That(property, Is.Not.Null, "the test runtime's System.Text.Json has AllowDuplicateProperties"); + var options = new JsonSerializerOptions().AddCultMathConverters(); + property!.SetValue(options, false); + + Action flat = () => JsonSerializer.Deserialize("""{"x":1,"x":2}""", options); + flat.Should().Throw(); + Action nested = () => JsonSerializer.Deserialize("""{"min":{"x":1,"x":2}}""", options); + nested.Should().Throw(); + JsonSerializer.Deserialize("""{"x":1,"y":2}""", options).Should().Be(new float2(1f, 2f)); + } + + [Test] + public void DictionaryKeys_ReadOnlyTheCanonicalSpelling() + { + foreach (var key in new[] { "1.0,2", "+1,2", " 1,2", "1, 2", "01,2", "1,2 ", "1e0,2", "1,2,", ",1" }) + { + Action read = () => Parse>($$"""{"{{key}}":1}"""); + read.Should().Throw(key); + } + + foreach (var key in new[] { "01,2", "+1,2", "1,-0" }) + { + Action read = () => Parse>($$"""{"{{key}}":1}"""); + read.Should().Throw(key); + } + + Action bools = () => Parse>("""{"True,false":1}"""); + bools.Should().Throw(); + Parse>("""{"1,2":1}""").Should().ContainKey(new float2(1f, 2f)); + } + + [Test] + public void NegativeZero_IsADistinctKey() + { + var negative = new float2(-0f, 0f); + KeyOf(negative).Should().Be("-0,0"); + KeyOf(float2.zero).Should().Be("0,0"); + KeyOf(new double2(0d, -0d)).Should().Be("0,-0"); + + var parsed = Parse>("""{"-0,0":1}""").Keys.Single(); + float.IsNegative(parsed.x).Should().BeTrue(); + var positive = Parse>("""{"0,0":1}""").Keys.Single(); + float.IsNegative(positive.x).Should().BeFalse(); + } + + [Test] + public void Culture_DoesNotChangeKeysOrValues() + { + var german = (CultureInfo)CultureInfo.InvariantCulture.Clone(); + german.NumberFormat.NumberDecimalSeparator = ","; + german.NumberFormat.NumberGroupSeparator = "."; + german.NumberFormat.NegativeSign = "−"; + german.NumberFormat.PositiveSign = "➕"; + var swedish = (CultureInfo)CultureInfo.InvariantCulture.Clone(); + swedish.NumberFormat.NegativeSign = "−"; + swedish.NumberFormat.NumberDecimalSeparator = ","; + + var original = CultureInfo.CurrentCulture; + try + { + foreach (var culture in new[] { german, swedish }) + { + CultureInfo.CurrentCulture = culture; + var options = With(ReadStrings | WriteStrings | Named); + + KeyOf(new float2(-1.5f, 2.25f)).Should().Be("-1.5,2.25"); + KeyOf(new double3(-1e-7, 1234.5, -0d)).Should().Be("-1E-07,1234.5,-0"); + KeyOf(new int2(-3, 4)).Should().Be("-3,4"); + KeyOf(new float2(float.NaN, float.NegativeInfinity), Named).Should().Be("NaN,-Infinity"); + Parse>("""{"-1.5,2.25":1}""").Keys.Single().Should().Be(new float2(-1.5f, 2.25f)); + Parse>("""{"-3,4":1}""").Keys.Single().Should().Be(new int2(-3, 4)); + + JsonSerializer.Serialize(new float2(-1.5f, 2.25f), options).Should().Be("""{"x":"-1.5","y":"2.25"}"""); + JsonSerializer.Deserialize("""{"x":"-1.5","y":"2.25"}""", options).Should().Be(new float2(-1.5f, 2.25f)); + JsonSerializer.Serialize(new int2(-3, 4), options).Should().Be("""{"x":"-3","y":"4"}"""); + JsonSerializer.Deserialize("""{"x":"-3","y":"4"}""", options).Should().Be(new int2(-3, 4)); + Json(new float2(-1.5f, 2.25f)).Should().Be("""{"x":-1.5,"y":2.25}"""); + } + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + private static void RoundTrip(T value) => Parse(Json(value)).Should().Be(value); } } From f6dbbf77ef49d1f2d0066d69af25276df95dcec2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:33:30 +0200 Subject: [PATCH 31/64] cultnet-rs: end the session of a peer whose packet the session refuses CultNetRudpSession::receive records a reliable sequence before reassembly, so keeping the session after it refuses a frame made the sender's retransmit be acknowledged and the frame silently lost. The hub and the socket transport now end that peer's session, send it a Disconnect, and surface the end to the caller (a Disconnected event, disconnect_reason()). The hub and socket constructors reject initial_sequence u32::MAX and max_pending_reliable_packets Some(0), so the accept_connect drops are unreachable. pending_accept_for_resend becomes public for the document server. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-rs/src/rudp.rs | 67 ++++++- .../cultnet-rs/tests/rudp_stray_packets.rs | 182 +++++++++++++----- 2 files changed, 196 insertions(+), 53 deletions(-) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 226f5c7b..f91838a2 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -271,7 +271,9 @@ impl CultNetRudpSession { self.pending_reliable.len() + self.queued_reliable.len() } - fn pending_accept_for_resend(&mut self, now_ms: u64) -> Option { + /// The unacknowledged Accept, stamped as just sent, for a peer that + /// repeats its Connect. `None` once the peer has acknowledged it. + pub fn pending_accept_for_resend(&mut self, now_ms: u64) -> Option { let pending = self .pending_reliable .values_mut() @@ -1399,6 +1401,10 @@ pub struct CultNetRudpServerHub { impl CultNetRudpServerHub { pub fn new(options: CultNetRudpServerHubOptions) -> Result { + validate_rudp_session_limits( + options.initial_sequence, + options.max_pending_reliable_packets, + )?; let local_addr = options.socket.local_addr()?; if options.max_peers == 0 { return Err(anyhow!( @@ -1589,7 +1595,7 @@ impl CultNetRudpServerHub { && peer.context.connect_payload == packet.payload { if peer.session.receive(&packet, now_ms()).is_err() { - self.stats.packets_dropped += 1; + self.end_refusing_peer(remote_addr)?; return Ok(true); } let reply = peer @@ -1650,7 +1656,7 @@ impl CultNetRudpServerHub { return Ok(true); }; let Ok(result) = peer.session.receive(&packet, now_ms()) else { - self.stats.packets_dropped += 1; + self.end_refusing_peer(remote_addr)?; return Ok(true); }; let context = peer.context.clone(); @@ -1696,6 +1702,25 @@ impl CultNetRudpServerHub { Ok(true) } + /// An admitted peer sent a packet its session refuses. `receive` has + /// already recorded the packet's reliable sequence, so the session cannot be + /// kept: a retransmit would be acknowledged and the frame silently lost. + /// The session ends, the peer is told, and the caller sees `Disconnected`. + fn end_refusing_peer(&mut self, remote_addr: SocketAddr) -> Result<()> { + self.stats.packets_dropped += 1; + let Some(mut peer) = self.peers.remove(&remote_addr) else { + return Ok(()); + }; + let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); + let goodbye = peer.session.create_disconnect(reason.clone()); + self.pending_events + .push_back(CultNetRudpServerEvent::Disconnected { + session: peer.context, + reason, + }); + self.send_packet(remote_addr, &goodbye) + } + pub fn poll_resends(&mut self) -> Result<()> { let now = now_ms(); let mut packets = Vec::new(); @@ -1743,6 +1768,10 @@ impl CultNetRudpServerHub { impl CultNetRudpSocketTransportConnection { pub fn new(options: CultNetRudpSocketTransportOptions) -> Result { + validate_rudp_session_limits( + options.initial_sequence, + options.max_pending_reliable_packets, + )?; let local_addr = options.socket.local_addr()?; let max_payload_bytes = options .max_payload_bytes @@ -2035,8 +2064,15 @@ impl CultNetRudpSocketTransportConnection { } let Ok(result) = self.session.receive(&packet, now_ms()) else { + // `receive` has already recorded the packet's reliable sequence, so + // the session cannot be kept: a retransmit would be acknowledged + // and the frame silently lost. End it and tell the peer. self.stats.packets_dropped += 1; - return Ok(true); + let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); + let goodbye = self.session.create_disconnect(reason.clone()); + self.disconnect_reason = Some(reason); + self.send_packet(&goodbye)?; + return Ok(false); }; if let Some(reply) = result.reply { self.send_packet(&reply)?; @@ -2181,6 +2217,29 @@ where } } +/// Sent to a peer whose session refused one of its packets, and reported to +/// the local caller as the disconnect reason. +const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; + +/// A session cannot admit a Connect when its sequence space starts exhausted or +/// its reliable queue holds nothing, so neither is a usable configuration. +fn validate_rudp_session_limits( + initial_sequence: u32, + max_pending_reliable_packets: Option, +) -> Result<()> { + if initial_sequence == u32::MAX { + return Err(anyhow!( + "RUDP initial_sequence must leave room for a reliable packet" + )); + } + if max_pending_reliable_packets == Some(0) { + return Err(anyhow!( + "RUDP max_pending_reliable_packets must be greater than zero" + )); + } + Ok(()) +} + #[derive(Clone, Debug, Default)] pub struct RudpTransportProfileOptions { pub transport_id: Option, diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 5dc39732..7a371e9e 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -273,46 +273,101 @@ per_stray!( server_mode_survives_foreign_connection_ids ); -#[test] -fn hub_drops_a_packet_its_admitted_peer_session_refuses() -> Result<()> { - let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( - "hub", - socket()?, - CONNECTION_ID, - ))?; - let hub_addr = hub.local_addr()?; - let peer = socket()?; - let mut peer_session = raw_session(CONNECTION_ID); - send_to(&peer, hub_addr, &peer_session.create_connect(0, b"peer".to_vec())?)?; - assert!(matches!( - hub.receive_event_once()?, - Some(CultNetRudpServerEvent::Connected { .. }) - )); - let mut buffer = vec![0_u8; 65_535]; - let (received, _) = peer.recv_from(&mut buffer)?; - peer_session.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; +/// A session that refuses a packet has already recorded its reliable sequence, +/// so keeping it would acknowledge the sender's retransmit and lose the frame. +/// The session ends and the sender is told. +fn refused_frame_options() -> (u32, Vec) { + (64, vec![7_u8; 100]) +} - send_to(&peer, hub_addr, &poisoned(&mut peer_session)?)?; - assert!(hub.receive_event_once()?.is_none()); - assert_eq!(hub.stats().packets_dropped, 1); - assert_eq!(hub.sessions().len(), 1); +fn client_with_fast_resend(server_addr: SocketAddr) -> Result { + let mut options = + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID); + options.resend_delay_ms = 1; + CultNetRudpSocketTransportConnection::new(options) +} - send_to( - &peer, - hub_addr, - &peer_session.send("schema", b"still here".to_vec(), reliable())?, - )?; - match hub.receive_event_once()? { - Some(CultNetRudpServerEvent::Frame { frame, .. }) => { - assert_eq!(frame.payload, b"still here"); - } - other => panic!("the peer must still be served, got {other:?}"), - } +/// The client's view after the server refused its frame: told the session is +/// over, and never shown the frame as acknowledged, even after it retransmits. +fn assert_client_sees_refusal( + client: &mut CultNetRudpSocketTransportConnection, + receipt: &CultNetRudpReliableSendReceipt, + mut serve: impl FnMut() -> Result<()>, +) -> Result<()> { + serve()?; + let _ = client.receive_once()?; + assert!( + client.disconnect_reason().is_some(), + "the client must be told the session ended" + ); + std::thread::sleep(Duration::from_millis(10)); + client.poll_resends()?; + serve()?; + let _ = client.receive_once()?; + assert_ne!( + client.reliable_send_status(receipt), + CultNetRudpReliableSendStatus::Acknowledged, + "a refused frame must never be acknowledged" + ); Ok(()) } #[test] -fn client_drops_a_packet_its_session_refuses() -> Result<()> { +fn hub_ends_the_session_of_a_peer_whose_reliable_frame_it_refuses() -> Result<()> { + let (limit, payload) = refused_frame_options(); + let mut options = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); + options.max_payload_bytes = Some(limit); + let mut hub = CultNetRudpServerHub::new(options)?; + let hub_addr = hub.local_addr()?; + let mut client = client_with_fast_resend(hub_addr)?; + client.connect(b"peer".to_vec())?; + assert!(matches!( + hub.receive_event_once()?, + Some(CultNetRudpServerEvent::Connected { .. }) + )); + let _ = client.receive_once()?; + assert!(client.connected()); + + let receipt = client.send_reliable("schema", payload)?; + let mut ended = false; + assert_client_sees_refusal(&mut client, &receipt, || { + while let Some(event) = hub.receive_event_once()? { + ended |= matches!(event, CultNetRudpServerEvent::Disconnected { .. }); + } + Ok(()) + })?; + assert!(ended, "the hub must report the ended session"); + assert_eq!(hub.sessions().len(), 0); + assert!(hub.stats().packets_dropped >= 1); + Ok(()) +} + +#[test] +fn server_mode_ends_the_session_of_a_peer_whose_reliable_frame_it_refuses() -> Result<()> { + let (limit, payload) = refused_frame_options(); + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut options = CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID); + options.max_payload_bytes = Some(limit); + let mut server = CultNetRudpSocketTransportConnection::new(options)?; + let mut client = client_with_fast_resend(server_addr)?; + client.connect(b"peer".to_vec())?; + let _ = server.receive_once()?; + let _ = client.receive_once()?; + assert!(client.connected()); + + let receipt = client.send_reliable("schema", payload)?; + assert_client_sees_refusal(&mut client, &receipt, || { + let _ = server.receive_once()?; + Ok(()) + })?; + assert!(server.disconnect_reason().is_some()); + assert!(server.stats().packets_dropped >= 1); + Ok(()) +} + +#[test] +fn client_ends_its_session_when_it_refuses_a_packet() -> Result<()> { let server = socket()?; let server_addr = server.local_addr()?; let mut client = CultNetRudpSocketTransportConnection::new( @@ -330,17 +385,17 @@ fn client_drops_a_packet_its_session_refuses() -> Result<()> { send_to(&server, client_addr, &poisoned(&mut server_session)?)?; assert!(client.receive_once()?.is_none()); assert_eq!(client.stats().packets_dropped, 1); - send_to( - &server, - client_addr, - &server_session.send("schema", b"real".to_vec(), reliable())?, - )?; - assert_eq!(client.receive_once()?.expect("the real frame").payload, b"real"); + assert!(client.disconnect_reason().is_some()); + + // The server is told, not left holding a live session. + let (received, _) = server.recv_from(&mut buffer)?; + let told = decode_rudp_packet(&buffer[..received])?; + assert_eq!(told.packet_type, CultNetRudpPacketType::Disconnect); Ok(()) } #[test] -fn server_mode_drops_what_is_not_its_peer_or_what_its_session_refuses() -> Result<()> { +fn server_mode_ends_the_session_when_the_session_refuses_a_packet() -> Result<()> { let server_socket = socket()?; let server_addr = server_socket.local_addr()?; let mut server = CultNetRudpSocketTransportConnection::new( @@ -367,16 +422,45 @@ fn server_mode_drops_what_is_not_its_peer_or_what_its_session_refuses() -> Resul assert!(server.receive_once()?.is_none()); assert_eq!(server.stats().packets_dropped, 6); - // A packet from the peer that the session itself refuses. + // A packet from the peer that the session itself refuses ends the session + // and the peer is told. send_to(&peer, server_addr, &poisoned(&mut peer_session)?)?; assert!(server.receive_once()?.is_none()); assert_eq!(server.stats().packets_dropped, 7); - - send_to( - &peer, - server_addr, - &peer_session.send("schema", b"real".to_vec(), reliable())?, - )?; - assert_eq!(server.receive_once()?.expect("the real frame").payload, b"real"); + assert!(server.disconnect_reason().is_some()); + let (received, _) = peer.recv_from(&mut buffer)?; + assert_eq!( + decode_rudp_packet(&buffer[..received])?.packet_type, + CultNetRudpPacketType::Disconnect + ); + Ok(()) +} + +#[test] +fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { + let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); + hub.initial_sequence = u32::MAX; + assert!(CultNetRudpServerHub::new(hub).is_err()); + let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); + hub.max_pending_reliable_packets = Some(0); + assert!(CultNetRudpServerHub::new(hub).is_err()); + + for build in [ + (|s, a| CultNetRudpSocketTransportOptions::client("c", s, a, CONNECTION_ID)) + as fn(UdpSocket, SocketAddr) -> CultNetRudpSocketTransportOptions, + |s, _| CultNetRudpSocketTransportOptions::server("s", s, CONNECTION_ID), + ] { + let addr = "127.0.0.1:9".parse()?; + let mut options = build(socket()?, addr); + options.initial_sequence = u32::MAX; + assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); + let mut options = build(socket()?, addr); + options.max_pending_reliable_packets = Some(0); + assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); + let mut options = build(socket()?, addr); + options.initial_sequence = u32::MAX - 1; + options.max_pending_reliable_packets = Some(1); + assert!(CultNetRudpSocketTransportConnection::new(options).is_ok()); + } Ok(()) } From f9034eedb0e49005d69977a6aaeeafb05076286d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:33:30 +0200 Subject: [PATCH 32/64] cultmesh-rs: a repeated Connect re-sends the pending Accept and queues nothing The document server called accept_connect again for a Connect on an existing key, queuing another reliable Accept each time: a resend storm, and the full queue Err that the previous commit turned into a drop. The repeat now re-sends the Accept still pending, or an Ack once acknowledged, as the hub does. A fresh session's accept_connect cannot fail under validated options, so its error is propagated again rather than counted as a drop. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 77 ++++++++++--------- .../cultmesh-rs/tests/rudp_document_server.rs | 20 +++-- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index d5e0b473..524d17c7 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -240,8 +240,8 @@ where } /// Datagrams read and discarded because they belong to no admissible - /// session: malformed frames, unknown sessions, a Connect the session or - /// the session cap refuses, a packet that fails its session. Never an + /// session: malformed frames, unknown sessions, a Connect the session cap + /// refuses, a packet that fails its session. Never an /// error: a moved flow or a scanner must not end the daemon loop. pub fn packets_dropped(&self) -> u64 { self.packets_dropped @@ -402,50 +402,53 @@ where }) } - /// `Ok(false)` when the Connect was refused (session cap, or the session - /// cannot queue another Accept); only a socket failure is an `Err`. + /// `Ok(false)` when the Connect was refused because the session cap is full; + /// only a socket failure is an `Err`. fn accept_connection( &mut self, key: CultMeshRudpSessionKey, packet: &CultNetRudpPacket, now: u64, ) -> Result { - if !self.sessions.contains_key(&key) { - if self.sessions.len() >= self.options.max_sessions { - return Ok(false); + let reply = match self.sessions.get_mut(&key) { + // The same key is a retransmitted Connect for the existing epoch: + // repeat the Accept still awaiting acknowledgement, or acknowledge + // if none is. It queues nothing, so a Connect storm cannot grow the + // session's reliable queue. A genuinely fresh client incarnation + // must choose a fresh id. + Some(entry) => { + entry.last_activity_monotonic_millis = now; + entry + .session + .pending_accept_for_resend(now) + .unwrap_or_else(|| entry.session.create_ack()) } - self.sessions.insert( - key, - SessionEntry { - session: CultNetRudpSession::new(CultNetRudpSessionOptions { - connection_id: key.connection_id, - initial_sequence: 1, - resend_delay_ms: duration_millis(self.options.resend_delay), - max_pending_reliable_packets: Some( - self.options.max_pending_reliable_packets_per_session, - ), - }), - created_at_monotonic_millis: now, - last_activity_monotonic_millis: now, - admitted_payload_bytes: 0, - }, - ); - } - - let accept = { - let entry = self - .sessions - .get_mut(&key) - .expect("accepted session must be present"); - // The same key is a retransmitted Connect for the existing epoch. - // A genuinely fresh client incarnation must choose a fresh id. - entry.last_activity_monotonic_millis = now; - match entry.session.accept_connect(packet, now, Vec::new()) { - Ok(accept) => accept, - Err(_) => return Ok(false), + None => { + if self.sessions.len() >= self.options.max_sessions { + return Ok(false); + } + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: key.connection_id, + initial_sequence: 1, + resend_delay_ms: duration_millis(self.options.resend_delay), + max_pending_reliable_packets: Some( + self.options.max_pending_reliable_packets_per_session, + ), + }); + let accept = session.accept_connect(packet, now, Vec::new())?; + self.sessions.insert( + key, + SessionEntry { + session, + created_at_monotonic_millis: now, + last_activity_monotonic_millis: now, + admitted_payload_bytes: 0, + }, + ); + accept } }; - self.send_packet(key.remote_addr, &accept)?; + self.send_packet(key.remote_addr, &reply)?; Ok(true) } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index e1e912e0..980bf04b 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -566,7 +566,7 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { max_pending_reliable_packets_per_session: 4, ..Default::default() }; - let mut server = server(options, clock, sink.clone(), Source::default())?; + let mut server = server(options, clock.clone(), sink.clone(), Source::default())?; let target = server.local_addr()?; let mut real = client(target, 101)?; connect(&mut server, &mut [&mut real])?; @@ -602,16 +602,22 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { send_raw(&mut server, &unadmitted.send("schema", vec![1], reliable.clone())?)?; assert_eq!(server.packets_dropped(), 2); - // A moved flow re-sends Connect from a socket that never hears the Accept: - // each one asks the session to queue one more unacknowledged Accept, and - // the fifth finds the queue full. + // A moved flow re-sends Connect from a socket that never hears the Accept. + // Each repeat re-sends the Accept still awaiting acknowledgement and queues + // nothing, so the storm neither errors nor grows the session's queue. let mut stormer = session(102); let connect_packet = stormer.create_connect(0, Vec::new())?; - for _ in 0..12 { + for _ in 0..200 { send_raw(&mut server, &connect_packet)?; } - assert_eq!(server.packets_dropped(), 2 + 8); + assert_eq!(server.packets_dropped(), 2); assert_eq!(server.session_count(), 2); + clock.set(60_000 + 1_000); + assert_eq!( + server.maintain()?.packets_resent, + 1, + "one Accept awaits acknowledgement, however many Connects repeated" + ); // A packet its own admitted session refuses ends that session, not the loop. stormer.assume_connected(0); @@ -619,7 +625,7 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { poison.fragment_count = 2; poison.fragment_id = 0; send_raw(&mut server, &poison)?; - assert_eq!(server.packets_dropped(), 2 + 8 + 1); + assert_eq!(server.packets_dropped(), 3); assert_eq!(server.session_count(), 1); send( From 5adbcaa0c8adf0992d27acfe67adf5de68b18434 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:34:54 +0200 Subject: [PATCH 33/64] Maps: record the C2a rollout-order and mark-by-content rulings, publication Cuts 1-2 landing Co-Authored-By: Claude Opus 5.5 --- docs/cultcache-publication-cut.md | 18 ++++++++++++++++-- docs/document-variants-cut.md | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/cultcache-publication-cut.md b/docs/cultcache-publication-cut.md index 7143127a..51953101 100644 --- a/docs/cultcache-publication-cut.md +++ b/docs/cultcache-publication-cut.md @@ -1,7 +1,21 @@ # CultCache publication: the cache owns "published once", and order stays data -Status: cut map, Imagination pass 1, 2026-09-29. It is not committed yet. Self -commits it to `main` as `docs/cultcache-publication-cut.md`. The ends are the +Status, 2026-09-30: Cuts 1-2 landed on `main` at `2a1b8a8`, from `hands/cultcache-pub-c1c2` at `5fb3fca`. +Soul verdict: merge. Suites green: Caching 323, Networking 295, Mesh 262. Cut 3 is in Hands on +`hands/cultcache-pub-c3`, with Soul's Cuts 1-2 follow-ups folded in: +- F4: pin `Held`'s publish-on-throw guard. This is needed before Cut 4 relies on sequence density; Stryker could + not mutate `Held` (CS0165). +- F3: pin observer removal on unsubscribe. +- F1: the "getters run once" claim is false for conditional commits on variant stores. + +Recorded: +- F2, the Q-P2 hazard. A rethrown observer exception makes a committed write look cancelled or refused. The + only live typed catch around a cache write is AetheriaEve `Aetheria.State.Daemon/Program.cs:3722`. It needs a + fail-fast handler, and no production code installs one. +- F5: subscribe and unsubscribe cost O(n). +- F6: writes after `Dispose` are dropped silently. + +Map written by Imagination pass 1, 2026-09-29. The ends are the operator ruling of 2026-09-30: every committed cache change reaches subscribers exactly once. This document owns the means. diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index 923c1735..a23fe4ea 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -339,6 +339,23 @@ Soul held it. Fixes are in Hands: - **C2b precondition (Soul F7).** Whether a pre-id element keeps its load-time id depends on the write path. C2b must refuse an override that targets a record whose ids exist only in memory, or mint on open. +**C2a Soul pass 2 (`e85b201b`), 2026-09-30: hold.** The tripwire only works against unreleased readers. +Every published release (`cultlib-unity-v1.0.60`, `caching-unity-v1.4.0`, `cultcache-ts-v0.14.0`, +`cultcache-py-v0.3.0`) predates C0. The 1.0.60 C# reader loads a v3 store, and its next write is v1 with the ids +stripped (probe). Other findings: +- F2: the same element object twice in one list commits, then the store refuses to load. +- F3: CultMesh single-file writes bypass the marker. +- F4: TS `pushAll` writes the header without reading it first. + +**C2a rulings, 2026-09-30 (operator):** +- **Rollout order closes the released-reader hole.** Ship a CultLib release that refuses v3, move every C# + reader of a store onto it, then let a C2a writer touch that store. There is no format change for this. + This supersedes the claim above that old readers are tripwired: that claim holds only for post-C0 readers. +- **Mark by content.** A store is written as v3 (v5 for directory stores) only when a record actually holds an + element id. An existing v3 header on disk stays. +- Fix batch 3 in Hands: F2, marking by content in every runtime, F3 routed through the cache's header + decision and load checks, F4. + **Operator rulings, 2026-09-30:** - **Duplicate unique-index values do not load** ("Duplicate indices should not load"). This covers plain records too, so R6-for-indexes becomes one general rule: From 1d0dc3aa57f3a26f89161dd6c9518f042a166b21 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:35:30 +0200 Subject: [PATCH 34/64] cultnet-rs: repeated Connect has one primitive; the goodbye must not ack the refused frame The Disconnect a refusing session sent carried the ack of the very frame it refused, so the sender saw the frame acknowledged. The session forgets its peer's sequences before building the goodbye, and a server-mode transport forgets its endpoint so only a Connect can claim it again. CultNetRudpSession::answer_repeated_connect is the one answer to a repeated Connect for the hub and the document server: it records the sequence (the document server's old accept_connect did, and a client that reconnects in one session depends on it), applies acks, queues nothing, and returns the pending Accept or an Ack. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 18 +++---- .../cultmesh-rs/tests/rudp_document_server.rs | 2 +- packages/cultnet-rs/src/rudp.rs | 49 +++++++++++++++---- .../cultnet-rs/tests/rudp_stray_packets.rs | 9 ++-- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 524d17c7..5536add0 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -411,17 +411,17 @@ where now: u64, ) -> Result { let reply = match self.sessions.get_mut(&key) { - // The same key is a retransmitted Connect for the existing epoch: - // repeat the Accept still awaiting acknowledgement, or acknowledge - // if none is. It queues nothing, so a Connect storm cannot grow the - // session's reliable queue. A genuinely fresh client incarnation - // must choose a fresh id. + // The same key is a retransmitted Connect for the existing epoch. + // A genuinely fresh client incarnation must choose a fresh id. Some(entry) => { entry.last_activity_monotonic_millis = now; - entry - .session - .pending_accept_for_resend(now) - .unwrap_or_else(|| entry.session.create_ack()) + match entry.session.answer_repeated_connect(packet, now) { + Ok(reply) => reply, + Err(_) => { + self.sessions.remove(&key); + return Ok(false); + } + } } None => { if self.sessions.len() >= self.options.max_sessions { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 980bf04b..d3ee1de8 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -581,7 +581,7 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { max_pending_reliable_packets: None, }) }; - let mut send_raw = |server: &mut Server, packet: &cultnet_rs::CultNetRudpPacket| -> Result<()> { + let send_raw = |server: &mut Server, packet: &cultnet_rs::CultNetRudpPacket| -> Result<()> { raw.send_to(&encode_rudp_packet(packet)?, target)?; server.poll_once()?; Ok(()) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index f91838a2..00e0e61b 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -271,9 +271,7 @@ impl CultNetRudpSession { self.pending_reliable.len() + self.queued_reliable.len() } - /// The unacknowledged Accept, stamped as just sent, for a peer that - /// repeats its Connect. `None` once the peer has acknowledged it. - pub fn pending_accept_for_resend(&mut self, now_ms: u64) -> Option { + fn pending_accept_for_resend(&mut self, now_ms: u64) -> Option { let pending = self .pending_reliable .values_mut() @@ -397,6 +395,30 @@ impl CultNetRudpSession { Ok(response) } + /// Answers a Connect from a peer this session has already accepted. The + /// packet's sequence is remembered and nothing is queued, so a Connect storm + /// cannot grow the reliable queue. The reply is the Accept still awaiting + /// acknowledgement, or an Ack once it was acknowledged. + pub fn answer_repeated_connect( + &mut self, + packet: &CultNetRudpPacket, + now_ms: u64, + ) -> Result { + self.require_connection(packet)?; + if packet.packet_type != CultNetRudpPacketType::Connect { + return Err(anyhow!( + "Expected RUDP connect packet, got {:?}", + packet.packet_type + )); + } + self.apply_acknowledgements(packet); + self.remember_received(packet.sequence); + self.last_received_at_ms = Some(now_ms); + Ok(self + .pending_accept_for_resend(now_ms) + .unwrap_or_else(|| self.create_ack())) + } + pub fn send( &mut self, channel_id: &str, @@ -1594,14 +1616,10 @@ impl CultNetRudpServerHub { if let Some(peer) = self.peers.get_mut(&remote_addr) && peer.context.connect_payload == packet.payload { - if peer.session.receive(&packet, now_ms()).is_err() { + let Ok(reply) = peer.session.answer_repeated_connect(&packet, now_ms()) else { self.end_refusing_peer(remote_addr)?; return Ok(true); - } - let reply = peer - .session - .pending_accept_for_resend(now_ms()) - .unwrap_or_else(|| peer.session.create_ack()); + }; self.send_packet(remote_addr, &reply)?; return Ok(true); } @@ -1712,6 +1730,9 @@ impl CultNetRudpServerHub { return Ok(()); }; let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); + // Forget the peer's sequences first: the goodbye's ack field would + // otherwise acknowledge the very frame the session refused. + peer.session.reset_peer_state(); let goodbye = peer.session.create_disconnect(reason.clone()); self.pending_events .push_back(CultNetRudpServerEvent::Disconnected { @@ -2068,10 +2089,18 @@ impl CultNetRudpSocketTransportConnection { // the session cannot be kept: a retransmit would be acknowledged // and the frame silently lost. End it and tell the peer. self.stats.packets_dropped += 1; + // The goodbye is built after the reset, or its ack field would + // acknowledge the very frame the session refused. A server-mode + // transport also forgets its peer: only a Connect can claim it again. let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); + self.session.reset_peer_state(); let goodbye = self.session.create_disconnect(reason.clone()); self.disconnect_reason = Some(reason); - self.send_packet(&goodbye)?; + let sent = self.send_packet(&goodbye); + if self.mode == CultNetRudpSocketMode::Server { + self.remote_addr = None; + } + sent?; return Ok(false); }; if let Some(reply) = result.reply { diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 7a371e9e..161773f6 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -388,9 +388,12 @@ fn client_ends_its_session_when_it_refuses_a_packet() -> Result<()> { assert!(client.disconnect_reason().is_some()); // The server is told, not left holding a live session. - let (received, _) = server.recv_from(&mut buffer)?; - let told = decode_rudp_packet(&buffer[..received])?; - assert_eq!(told.packet_type, CultNetRudpPacketType::Disconnect); + let mut told = false; + while let Ok((received, _)) = server.recv_from(&mut buffer) { + told |= decode_rudp_packet(&buffer[..received])?.packet_type + == CultNetRudpPacketType::Disconnect; + } + assert!(told, "the server must be told the session ended"); Ok(()) } From be91ea837a1318ba85198c74068b03f151843d34 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:36:41 +0200 Subject: [PATCH 35/64] Pin that a failed hold publishes what it minted and an unsubscribed observer stops A hold whose body throws after it admitted changes still publishes them, and Held now assigns its result up front so it compiles under Stryker Safe Mode (CS0165 had dropped every mutant in it). Disposing a Watch subscription removes the observer from the cache. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 2 +- .../CultCacheObserverTests.cs | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 245e9367..d9e8c2f8 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2391,7 +2391,7 @@ namespace GameCult.Caching throw new InvalidOperationException( "A cache hold was entered inside another cache's hold; a thread holds one cache's gate at a time."); var mine = _held = new List<(Change Change, bool Loaded)>(); - T result; + var result = default(T)!; try { lock (_gate) diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs index 7fde01e7..fa87ea4f 100644 --- a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -122,6 +122,61 @@ namespace GameCult.Caching.Tests }); } + [Test] + public void AnUnsubscribedObserverStopsReceiving() + { + using var cache = new CultCache(Registry); + var seen = new List(); + var subscription = cache.Watch().Subscribe(change => seen.Add(change.Document!.Text)); + Send(cache, "one"); + + subscription.Dispose(); + Send(cache, "two"); + + Assert.That(seen, Is.EqualTo(new[] { "one" })); + } + + // A store that admits a load and then fails: the hold's body throws after the changes were minted. + private sealed class LoadThenFailStore : CacheBackingStore + { + public bool Armed; + + public override void PullAll() => Held(() => + { + if (!Armed) + return; + var record = new CultStoredDocument( + KeyOf("loaded"), "2026-01-01T00:00:00.0000000+00:00", Registry.GetRequired(typeof(ObservedPing)), new ObservedPing { Text = "loaded" }); + Loaded!(new[] { record }, Array.Empty()); + throw new InvalidOperationException("store failed after the load"); + }); + + public override void Push(CultStoredDocument entry) { } + public override void Delete(CultStoredDocument entry) { } + public override void PushAll() { } + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => CultCommitOutcome.Committed; + } + + [Test] + public void AHoldWhoseBodyThrowsStillPublishesTheChangesItMinted() + { + using var cache = new CultCache(Registry); + var store = new LoadThenFailStore(); + cache.AddBackingStore(store); + var seen = new List(); + using var subscription = cache.Watch().Subscribe(change => seen.Add(change.Sequence)); + store.Armed = true; + + var thrown = Assert.Throws(() => store.PullAll())!; + + Assert.Multiple(() => + { + Assert.That(thrown.Message, Is.EqualTo("store failed after the load"), "the body's own exception wins"); + Assert.That(cache.Get(KeyOf("loaded")), Is.Not.Null, "the load landed"); + Assert.That(seen, Is.EqualTo(new[] { 1L }), "and reached its observers"); + }); + } + [Test] public void DisposeCompletesObserversAndACompletedCacheCompletesNewOnes() { From 384ee30ff6cc8883eb1e4e2d82cbffafeb3d2d18 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:37:03 +0200 Subject: [PATCH 36/64] Say that a conditional variant commit reads the getters a second time Judging the merge against the file re-runs Project for the records the commit lands and for re-resolved variants, so "once per landing record" holds only for the admission. Handing the plan's projections to Judging would not cover the re-resolved dependents, so the contract now states the second read. Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-store-composition.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 6cf6cdfa..f42c00f7 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -83,10 +83,13 @@ which route wrote it. exception. The subscriber stays subscribed and the commit stands. A throwing handler cannot undo the store's adoption of a load; the store and cache already agree when publication starts. -- A record's `[CultName]` and `[CultIndex]` getters run once per landing record, - before the store is written and before any `Sequence` is minted. A getter that - throws refuses the whole change set, loads included: nothing is written, - minted or left in memory. +- An admission runs a record's `[CultName]` and `[CultIndex]` getters once per + landing record, before the store is written and before any `Sequence` is + minted. A conditional commit on a store that holds variants is judged a second + time against the file it merges onto, which runs the getters again for the + records it lands and for any variant it re-resolves; that judgement is also + before the store is written. A getter that throws refuses the whole change + set, loads included: nothing is written, minted or left in memory. - Pulling all stores pulls every attached store even if a handler throws during one store's load, then rethrows. - Hydration failure on open is loud: a corrupt store file, or a record whose From 610b5c86c23d7ff4b4fb4aac565624bfe9d42905 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:37:05 +0200 Subject: [PATCH 37/64] cultnet-ts: end a refusing session, survive operation-service rejections, cap its sessions A session that refuses a packet has already recorded its reliable sequence, so the socket transport now ends it: the session forgets the peer (so the goodbye does not acknowledge the refused frame), sends a Disconnect, forgets the endpoint in server mode, and emits disconnect and close. The operation service did `void handleServerDatagram(...)`, so any rejection was an unhandled rejection that ended Node; it now drops and counts the packet (packetsDropped), ends a session that refuses a frame, and holds at most 64 sessions, dropping a Connect from a new peer beyond that. CultNetRudpSession rejects an initialSequence of 0xFFFFFFFF. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/operation-service.ts | 43 ++++-- packages/cultnet-ts/src/rudp.ts | 26 ++++ packages/cultnet-ts/test/cultnet.test.ts | 135 +++++++++++++++++++ 3 files changed, 195 insertions(+), 9 deletions(-) diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index ee13457f..3ba3634e 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -17,6 +17,8 @@ import { import { CultNetPeer } from "./peer"; const DEFAULT_CONNECTION_ID = 0x43554c54; +/** Matches the document server's default session cap. A Connect past it is dropped. */ +const MAX_OPERATION_SESSIONS = 64; export interface CultNetOperationServerOptions { runtimeId: string; @@ -30,6 +32,8 @@ export interface CultNetOperationServerOptions { export interface CultNetOperationServer { readonly endpoint: string; + /** Datagrams read and discarded: malformed, unadmitted, refused by their session, or failed in handling. */ + readonly packetsDropped: number; close(): Promise; } @@ -57,8 +61,15 @@ export async function startCultNetOperationServer( const wire = encodeRudpPacket(packet); socket.send(wire, remote.port, remote.address); }; + let packetsDropped = 0; socket.on("message", (wire, remote) => { - void handleServerDatagram(socket, sessions, connectionId, options, wire, remote, sendPacket); + // What a datagram carries is the sender's business: a rejection here must + // drop and count the packet, never become an unhandled rejection that ends + // the process. + handleServerDatagram(sessions, connectionId, options, wire, remote, sendPacket).then( + admitted => { if (!admitted) packetsDropped += 1; }, + () => { packetsDropped += 1; }, + ); }); await bindSocket(socket, options.port ?? 0, options.host ?? "127.0.0.1"); const resendTimer = setInterval(() => { @@ -71,6 +82,7 @@ export async function startCultNetOperationServer( const endpoint = `rudp://${address.address}:${address.port}`; return { endpoint, + get packetsDropped() { return packetsDropped; }, close: async () => { clearInterval(resendTimer); await closeSocket(socket); @@ -118,37 +130,49 @@ export async function invokeCultNetOperation( } } +/** Resolves false when the datagram was dropped without an error. */ async function handleServerDatagram( - socket: Socket, sessions: Map, connectionId: number, options: CultNetOperationServerOptions, wire: Buffer, remote: RemoteInfo, sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, -): Promise { +): Promise { let packet: CultNetRudpPacket; try { packet = decodeRudpPacket(wire); } catch { - return; + return false; } - if (packet.connectionId !== connectionId) return; + if (packet.connectionId !== connectionId) return false; const key = `${remote.address}:${remote.port}`; let peer = sessions.get(key); if (packet.packetType === "connect") { + if (!peer && sessions.size >= MAX_OPERATION_SESSIONS) return false; peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote }; sessions.set(key, peer); sendPacket(remote, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); - return; + return true; + } + if (!peer) return false; + let result: ReturnType; + try { + result = peer.session.receive(packet, Date.now()); + } catch { + // receive() has already recorded the packet's reliable sequence, so the + // session cannot be kept: end it and tell the peer. The reset comes first, + // or the goodbye's ack field would acknowledge the refused frame. + sessions.delete(key); + peer.session.resetPeerState(); + sendPacket(remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + return false; } - if (!peer) return; - const result = peer.session.receive(packet, Date.now()); if (result.reply) sendPacket(remote, result.reply); for (const ready of result.readyToSend ?? []) sendPacket(remote, ready); if (result.disconnected) { sessions.delete(key); - return; + return true; } for (const frame of result.delivered) { if (frame.channelId !== "schema") continue; @@ -166,6 +190,7 @@ async function handleServerDatagram( if (packet.packetType === "data" || result.delivered.length > 0) { sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); } + return true; } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 89e478c2..0f1cbc33 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -170,6 +170,9 @@ export class CultNetRudpSession { constructor(options: CultNetRudpSessionOptions) { this.connectionId = toUint32(options.connectionId, "connectionId"); this.#nextSequence = toUint32(options.initialSequence ?? 1, "initialSequence"); + if (this.#nextSequence === 0xffff_ffff) { + throw new Error("RUDP initialSequence must leave room for a reliable packet."); + } this.resendDelayMs = options.resendDelayMs ?? 250; if (options.maxPendingReliablePackets !== undefined && options.maxPendingReliablePackets <= 0) { throw new Error("RUDP maxPendingReliablePackets must be greater than zero."); @@ -921,6 +924,25 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } } + #endRefusedSession(): void { + const reason = Buffer.from("session refused a packet", "utf8"); + // The goodbye is built after the reset, or its ack field would acknowledge + // the very frame the session refused. + this.#session.resetPeerState(); + try { + this.#sendPacket(this.#session.createDisconnect(reason)); + } catch { + // Best-effort: the session ends whether or not the peer hears it. + } + if (this.#mode === "server") { + // Only a Connect can claim the endpoint again. + this.#remoteHost = undefined; + this.#remotePort = undefined; + } + this.emit("disconnect", { reason }); + this.emit("close"); + } + #receiveDatagram(wire: Buffer, remote: RemoteInfo): void { this.#stats.bytesReceived += wire.length; // What a datagram carries is the sender's business, not a fault of this @@ -975,7 +997,11 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement try { result = this.#session.receive(packet, Date.now()); } catch { + // receive() has already recorded the packet's reliable sequence, so the + // session cannot be kept: a retransmit would be acknowledged and the + // frame silently lost. End it and tell the peer. this.#stats.packetsDropped += 1; + this.#endRefusedSession(); return; } try { diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 735418e0..b9aaa2a5 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -38,6 +38,9 @@ import { createRudpTransportProfile, defineCultNetDocumentBinding, decodeRudpPacket, + type CultNetOperationRequestMessage, + type CultNetOperationResponseMessage, + type CultNetRudpPacket, encodeCultNetMessageForWire, encodeRudpPacket, ghostlightAgentStateGeneratedContract, @@ -1930,3 +1933,135 @@ test("rudp socket transport drops strays without an error event and keeps servin server.close(); } }); + +/** A reliable frame no session accepts: a fragment set with no fragment id. */ +function poisonedFrame(session: CultNetRudpSession): CultNetRudpPacket { + const [packet] = session.sendMany("schema", Buffer.from("poison"), { reliable: true, ordered: true, nowMs: 0 }); + return { ...packet!, fragmentCount: 2, fragmentId: 0, fragmentIndex: 0 }; +} + +test("rudp socket transport ends the session that refuses a reliable frame and never acknowledges it", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x10203051; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const peer = new CultNetRudpSession({ connectionId }); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + const disconnects: string[] = []; + server.on("disconnect", ({ reason }) => disconnects.push(Buffer.from(reason).toString("utf8"))); + + try { + toServer(peer.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the Accept"); + peer.receive(received.find((p) => p.packetType === "accept")!, 0); + + const poison = poisonedFrame(peer); + toServer(poison); + await waitFor(() => disconnects.length === 1, "the server ending the session"); + await waitFor(() => received.some((p) => p.packetType === "disconnect"), "the goodbye"); + assert.equal(server.stats.packetsDropped, 1); + assert.equal(server.connected, false); + + // The peer sees the session end and never sees the frame acknowledged, + // neither in the goodbye nor after it retransmits. + const goodbye = received.find((p) => p.packetType === "disconnect")!; + assert.equal(peer.receive(goodbye, 1).disconnected, true); + assert.ok(peer.pendingReliableSequences.includes(poison.sequence), "the refused frame was acknowledged"); + const before = received.length; + toServer(poison); + await waitFor(() => server.stats.packetsDropped === 2, "the retransmit dropped"); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(received.length, before, "the server must not answer a retransmit for an ended session"); + + // Only a Connect claims the endpoint again. + const again = new CultNetRudpSession({ connectionId }); + toServer(again.createConnect(0)); + await waitFor(() => received.length > before, "the new Accept"); + assert.ok(received.slice(before).some((p) => p.packetType === "accept")); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("rudp sessions reject sequence spaces and queues that cannot admit a Connect", () => { + assert.throws(() => new CultNetRudpSession({ connectionId: 1, initialSequence: 0xffff_ffff }), /initialSequence/); + assert.throws(() => new CultNetRudpSession({ connectionId: 1, maxPendingReliablePackets: 0 }), /greater than zero/); + assert.doesNotThrow(() => new CultNetRudpSession({ connectionId: 1, initialSequence: 0xffff_fffe, maxPendingReliablePackets: 1 })); +}); + +test("operation service drops and counts a packet its session refuses and keeps serving", async () => { + const handler = (request: CultNetOperationRequestMessage): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }); + const server = await startCultNetOperationServer({ runtimeId: "sai-sidecar", handler }); + const raw = await bindUdpSocket(); + const port = Number(new URL(server.endpoint).port); + const connectionId = 0x43554c54; + const received: CultNetRudpPacket[] = []; + raw.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const peer = new CultNetRudpSession({ connectionId }); + const toServer = (packet: CultNetRudpPacket) => raw.send(encodeRudpPacket(packet), port, "127.0.0.1"); + try { + toServer(peer.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the Accept"); + peer.receive(received[0]!, 0); + toServer(poisonedFrame(peer)); + await waitFor(() => server.packetsDropped === 1, "the poisoned frame dropped"); + await waitFor(() => received.some((p) => p.packetType === "disconnect"), "the session ended"); + + const response = await invokeCultNetOperation(server.endpoint, { + schemaVersion: "cultnet.operation_request.v0", + messageId: "after-poison", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }, { runtimeId: "eve-test" }); + assert.equal(response.messageId, "after-poison"); + } finally { + raw.close(); + await server.close(); + } +}); + +test("operation service drops a Connect once its session table is full", async () => { + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: () => { throw new Error("no requests expected"); }, + }); + const port = Number(new URL(server.endpoint).port); + const sockets: Socket[] = []; + try { + for (let index = 0; index < 66; index += 1) { + const socket = await bindUdpSocket(); + sockets.push(socket); + socket.send(encodeRudpPacket(new CultNetRudpSession({ connectionId: 0x43554c54 }).createConnect(0)), port, "127.0.0.1"); + } + await waitFor(() => server.packetsDropped === 2, "the Connects past the cap dropped"); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(server.packetsDropped, 2); + } finally { + for (const socket of sockets) socket.close(); + await server.close(); + } +}); From 042dc75c82de77207cefdbb8ad48f86dc59119d9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:39:35 +0200 Subject: [PATCH 38/64] cultnet-py: a server-mode transport is claimed only by a Connect; a refusing session ends Server mode claimed the endpoint on the first datagram of any type with the right connection id, so an orphan Ack made a later real client a stranger. Only a Connect claims it now. A session that refuses a packet has already recorded its reliable sequence, so the transport ends it: the session forgets the peer (so the goodbye does not acknowledge the refused frame), a Disconnect goes to the peer, disconnect_reason is set, and a server-mode transport forgets its endpoint. CultNetRudpSession gains reset_peer_state and rejects an initial_sequence of 0xFFFFFFFF. Co-Authored-By: Claude Opus 5.5 --- .../cultnet-py/src/cultnet_py/transport.py | 40 +++++++ packages/cultnet-py/tests/test_cultnet.py | 113 +++++++++++++++++- 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 14c33fcb..3462b047 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -410,6 +410,8 @@ class CultNetRudpSession: raise ValueError("RUDP max_pending_reliable_packets must be greater than zero") self.max_pending_reliable_packets = options.max_pending_reliable_packets self._next_sequence = _uint32(options.initial_sequence, "initial_sequence") + if self._next_sequence == 0xFFFFFFFF: + raise ValueError("RUDP initial_sequence must leave room for a reliable packet") self._next_sequenced_by_channel: dict[str, int] = {} self._next_fragment_id = 1 self._connected = False @@ -453,6 +455,21 @@ class CultNetRudpSession: def outstanding_reliable_packet_count(self) -> int: return len(self._pending_reliable) + len(self._queued_reliable) + def reset_peer_state(self) -> None: + """Forgets everything learned from the peer; sequence numbers already issued stay issued.""" + self._connected = False + self._last_received_at_ms = None + self._highest_received_sequence = None + self._received_sequences.clear() + self._next_sequenced_by_channel.clear() + self._latest_sequenced_by_channel.clear() + self._pending_reliable.clear() + self._queued_reliable.clear() + self._ordered_next_sequence_by_channel.clear() + self._ordered_buffers.clear() + self._fragment_buffers.clear() + self._fragment_sets_evicted = 0 + def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: self._ensure_reliable_capacity(1) packet = self._create_packet( @@ -997,6 +1014,10 @@ class CultNetRudpSocketTransportConnection: self._packets_dropped += 1 return None if self.remote_addr is None: + # Only a Connect claims the endpoint of a server-mode transport. + if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type != CultNetRudpPacketType.CONNECT: + self._packets_dropped += 1 + return None self.remote_addr = remote_addr elif remote_addr != self.remote_addr: self._packets_dropped += 1 @@ -1014,7 +1035,11 @@ class CultNetRudpSocketTransportConnection: try: result = self.session.receive(packet, _now_ms()) except ValueError: + # receive() has already recorded the packet's reliable sequence, so the + # session cannot be kept: a retransmit would be acknowledged and the + # frame silently lost. End it and tell the peer. self._packets_dropped += 1 + self._end_refused_session() return None if result.reply is not None: self._send_packet(result.reply) @@ -1089,6 +1114,21 @@ class CultNetRudpSocketTransportConnection: self._closed = True self.socket.close() + def _end_refused_session(self) -> None: + reason = b"session refused a packet" + # The goodbye is built after the reset, or its ack field would acknowledge + # the very frame the session refused. + self.session.reset_peer_state() + goodbye = self.session.create_disconnect(reason) + self.disconnect_reason = reason + try: + self._send_packet(goodbye) + except OSError: + pass # Best effort: the session ends whether or not the peer hears it. + if self.mode == CultNetRudpSocketMode.SERVER: + # Only a Connect can claim the endpoint again. + self.remote_addr = None + def _send_packet(self, packet: CultNetRudpPacket) -> None: if self.remote_addr is None: raise ValueError("RUDP socket transport does not have a remote endpoint") diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 549d0803..5c2680ef 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -8,7 +8,7 @@ import unittest import hashlib import hmac import io -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Callable @@ -977,6 +977,117 @@ class CultNetTests(unittest.TestCase): client.close() server.close() + def test_cultnet_rudp_server_mode_is_claimed_only_by_a_connect(self) -> None: + server_socket = bind_udp_socket() + orphan_socket = bind_udp_socket() + client_socket = bind_udp_socket() + connection_id = 0x10203052 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + ) + ) + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-client", + socket=client_socket, + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=server_socket.getsockname(), + connection_id=connection_id, + ) + ) + orphan = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) + try: + # A right-id packet that is not a Connect must claim nothing. + orphan_socket.sendto(encode_rudp_packet(orphan.create_ack()), server_socket.getsockname()) + self.assertIsNone(server.receive_once()) + self.assertEqual(server.stats.packets_dropped, 1) + self.assertIsNone(server.remote_addr) + + client.connect(b"join") + pump_rudp_handshake(client, server) + self.assertTrue(client.connected) + self.assertEqual(server.remote_addr, client_socket.getsockname()) + finally: + orphan_socket.close() + client.close() + server.close() + + def test_cultnet_rudp_socket_transport_ends_the_session_that_refuses_a_reliable_frame(self) -> None: + server_socket = bind_udp_socket() + peer_socket = bind_udp_socket() + connection_id = 0x10203053 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + ) + ) + peer = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) + + def to_server(packet) -> None: + peer_socket.sendto(encode_rudp_packet(packet), server_socket.getsockname()) + + def drain_peer() -> list: + packets = [] + while True: + try: + wire, _ = peer_socket.recvfrom(65535) + except TimeoutError: + return packets + packets.append(decode_rudp_packet(wire)) + + try: + to_server(peer.create_connect(0, b"join")) + server.receive_once() + accept = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0] + peer.receive(accept, 0) + + (frame,) = peer.send_many("schema", b"poison", CultNetRudpSendOptions(reliable=True, ordered=True)) + poison = replace(frame, fragment_count=2, fragment_id=0) + to_server(poison) + self.assertIsNone(server.receive_once()) + self.assertEqual(server.stats.packets_dropped, 1) + self.assertFalse(server.connected) + self.assertEqual(server.disconnect_reason, b"session refused a packet") + + # The peer is told, and the goodbye does not acknowledge the refused frame. + (goodbye,) = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.DISCONNECT] + self.assertTrue(peer.receive(goodbye, 1).disconnected) + self.assertIn(poison.sequence, peer.pending_reliable_sequences) + + # A retransmit is dropped, not acknowledged. + to_server(poison) + self.assertIsNone(server.receive_once()) + self.assertEqual(server.stats.packets_dropped, 2) + self.assertEqual(drain_peer(), []) + self.assertIn(poison.sequence, peer.pending_reliable_sequences) + + # A Connect claims the endpoint again. + again = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) + to_server(again.create_connect(0, b"again")) + server.receive_once() + self.assertEqual( + [p.packet_type for p in drain_peer()], [CultNetRudpPacketType.ACCEPT] + ) + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_session_rejects_limits_that_cannot_admit_a_connect(self) -> None: + with self.assertRaises(ValueError): + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=1, initial_sequence=0xFFFFFFFF)) + with self.assertRaises(ValueError): + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=1, max_pending_reliable_packets=0)) + CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=1, initial_sequence=0xFFFFFFFE, max_pending_reliable_packets=1) + ) + def test_cultnet_rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames(self) -> None: server_socket = bind_udp_socket() client_socket = bind_udp_socket() From 44ad5e89488c9dbfba35acfae8dd42649cb84860 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:40:02 +0200 Subject: [PATCH 39/64] Journal admissions under the cache gate, in Sequence order CultCache.AddJournal registers an ordered admission journal. It is called under the gate right after Apply, once per admission that changed something, with the admission's changes (dependents included) in Sequence order, before any observer. A journal that throws joins the hold's failures: the admission stands, the other journals run, the change is published and the writer hears of it afterwards. Any hold entered while a journal runs throws. A hold now carries its own changes and failures. Order stays data; no delivery thread or scheduler is added. Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-store-composition.md | 21 ++ src/GameCult.Caching/CultCache.cs | 84 ++++- .../CultCacheJournalTests.cs | 304 ++++++++++++++++++ 3 files changed, 400 insertions(+), 9 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/CultCacheJournalTests.cs diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index f42c00f7..251a1c1e 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -74,6 +74,22 @@ which route wrote it. - Not stale-protected: schema-alias handles (`AsSchemaAlias`) get no `Sequence`; removals do not advance a mirror's applied `Sequence`, and the mirrors do not surface removals. +- `AddJournal` registers an ordered admission journal, for audit, replication or + a derived index. The cache calls each journal under its gate, right after an + admission is applied and before that admission is published, once per + admission that changed something, with that admission's changes (a variant + re-resolved by a base edit included) in `Sequence` order. Journals of one cache + run in registration order, and reading the cache from one sees the new state. + Because the gate serializes admissions, the calls of one cache see contiguous, + strictly increasing `Sequence` values: a journal is the one place where cache + order is a fact, and delivery order is not. A refused admission, or one a store + declines, is not journaled. Disposing the registration removes the journal. +- A journal that throws does not undo the admission: the remaining journals + still run, the change is published, and the exception is rethrown to the + writer after publication, with the same aggregation as an observer's. The + journal stays registered. Any hold on the cache while a journal runs throws + `InvalidOperationException`: a journal must not write to the cache or + register another journal. - An `OnUpdate` handler exception is rethrown to that caller after all of that call's changes are delivered (an `AggregateException` if several threw). If the call itself failed, its own exception is rethrown and handler exceptions @@ -103,6 +119,11 @@ which route wrote it. cache takes that cache's gate as its own lock, so a store's load callback into the cache cannot take the two out of order. A store must not call `Loaded` while holding its own lock outside the cache's hold. +- A journal runs under the cache's gate, so a slow journal slows every writer of + that cache. It must not enter the cache, and must not wait for anything that + needs the cache or its gate: another thread's write, a lock a writer holds. + Its own leaf locks, taken only inside the journal, are the only locks that may + nest under the gate. - A thread holds one cache's gate at a time: entering a cache's hold inside another cache's hold throws. - A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index d9e8c2f8..986ea974 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1813,8 +1813,11 @@ namespace GameCult.Caching private readonly object _observerGate = new(); private readonly object _gate = new(); private long _sequence; - // The changes admitted by this thread's outermost hold, published by that hold when it exits. - [ThreadStatic] private static List<(Change Change, bool Loaded)>? _held; + // What this thread's outermost hold admitted, published by that hold when it exits. + [ThreadStatic] private static Hold? _held; + // Journals run under the gate, in registration order; both fields are read and written only under it. + private readonly List>>> _journals = new(); + private bool _journaling; private bool _dirtyInMemory; // The codec is the store's encoding (CultCacheMessagePack.Create supplies it). A cache without one holds no variants: @@ -1924,6 +1927,23 @@ namespace GameCult.Caching return Watch().Where(change => change.Key.Equals(key)); } + // An ordered admission journal: called under the gate, right after an admission is applied, once per admission + // that changed something, with that admission's changes in Sequence order. Journals of one cache run in + // registration order. A journal sees the new state (reads work) and is the one place cache order is a fact: it + // must not write to the cache or wait for anything that needs it, and any hold on this cache while a journal runs + // throws. A journal's exception does not undo the admission: it is published, then the exception is rethrown to + // the writer, as an observer's is. + public IDisposable AddJournal(Action>> journal) + { + if (journal == null) throw new ArgumentNullException(nameof(journal)); + Held(() => + { + _journals.Add(journal); + return true; + }); + return Disposable.Create(() => Held(() => _journals.Remove(journal))); + } + // Attaching reads the store; there is no interval in which it is attached but unread. public void AddBackingStore(CacheBackingStore store, params Type[] homes) { @@ -2372,25 +2392,33 @@ namespace GameCult.Caching var outcome = land(home, plan.Admitted); if (outcome != CultCommitOutcome.Committed) return outcome; - foreach (var change in Apply(plan, evicted, source)) - _held.Add((change, source != null)); + var applied = Apply(plan, evicted, source); + foreach (var change in applied) + _held.Changes.Add((change, source != null)); + RunJournals(applied, _held.Failures); return CultCommitOutcome.Committed; }); // Every admission runs in a hold. A nested hold on the same cache adds to the outermost one, which publishes // exactly its own changes after it leaves the gate, before it returns, on its caller's thread: observers never // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not - // guaranteed; each change carries the Sequence it was admitted with. Every change reaches every observer even + // guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate, + // in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even // if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown. // If the body threw, its exception wins. internal T Held(Func body) { if (Monitor.IsEntered(_gate)) + { + if (_journaling) + throw new InvalidOperationException("A journal entered the cache it journals; a journal must not write to it."); return body(); + } + if (_held != null) throw new InvalidOperationException( "A cache hold was entered inside another cache's hold; a thread holds one cache's gate at a time."); - var mine = _held = new List<(Change Change, bool Loaded)>(); + var mine = _held = new Hold(); var result = default(T)!; try { @@ -2405,7 +2433,8 @@ namespace GameCult.Caching } _held = null; - var failures = Publish(mine); + var failures = mine.Failures; + failures.AddRange(Publish(mine)); if (failures.Count == 1) ExceptionDispatchInfo.Capture(failures[0]).Throw(); if (failures.Count > 1) @@ -2413,10 +2442,10 @@ namespace GameCult.Caching return result; } - private List Publish(List<(Change Change, bool Loaded)> changes) + private List Publish(Hold hold) { var failures = new List(); - foreach (var (change, loaded) in changes) + foreach (var (change, loaded) in hold.Changes) { // R3 routes a throwing subscriber to its unhandled-exception handler, and a fail-fast handler rethrows: every // observer still runs, the exception joins the failures, and the observer stays subscribed. @@ -2447,6 +2476,36 @@ namespace GameCult.Caching return failures; } + // Runs under the gate. A journal that throws joins the hold's failures and the remaining journals still run. + private void RunJournals(List applied, List failures) + { + if (_journals.Count == 0 || applied.Count == 0) + return; + var changes = applied + .Select(change => new CultCacheDocumentChange( + change.Kind, change.Stored.Key, change.Document, change.Previous, change.Sequence)) + .ToArray(); + _journaling = true; + try + { + foreach (var journal in _journals) + { + try + { + journal(changes); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + } + finally + { + _journaling = false; + } + } + private CacheBackingStore? Validate( IReadOnlyList admitted, IReadOnlyList evicted, @@ -2992,6 +3051,13 @@ namespace GameCult.Caching public CultRecordKey Key { get; } } + // What one outermost hold admitted, and what its journals threw. + private sealed class Hold + { + public List<(Change Change, bool Loaded)> Changes { get; } = new(); + public List Failures { get; } = new(); + } + private sealed class Change { public Change(CultCacheDocumentChangeKind kind, CultStoredDocument stored, object? document, object? previous, long sequence) diff --git a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs new file mode 100644 index 00000000..563669e0 --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // Cut 3 of docs/cultcache-publication-cut.md: a journal is called under the cache gate, right after an admission is + // applied, once per admission, with that admission's changes in Sequence order. It is the one place cache order is a fact. + [NonParallelizable] + public class CultCacheJournalTests + { + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(JournalNote) }); + + private string _directory = string.Empty; + + [SetUp] + public void Prepare() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-journal-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + JournalNote.Armed = false; + } + + [TearDown] + public void Cleanup() + { + JournalNote.Armed = false; + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + [CultDocument("tests.journal_note", "tests.journal_note.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class JournalNote + { + public static volatile bool Armed; + [IgnoreMember] private string _name = string.Empty; + + [Key(0)] + [CultName] + public string Name + { + get + { + if (Armed && _name == "boom") + throw new InvalidOperationException("name getter boom"); + return _name; + } + set => _name = value; + } + + [Key(1)] + public int Power; + } + + private static CultRecordKey KeyOf(string text) => new($"journal:{text}"); + + private static CultCache Memory() => new(Registry, CultCacheMessagePack.CreateCodec(Registry)); + + private static void Put(CultCache cache, string name, int power = 1) => + cache.Commit(batch => batch.Upsert(typeof(JournalNote), new JournalNote { Name = name, Power = power }, KeyOf(name))); + + [Test] + public void EveryAdmissionIsJournaledInSequenceOrderAndItsChangesArriveTogether() + { + using var cache = Memory(); + var calls = new List(); + using var journal = cache.AddJournal(changes => calls.Add(changes.Select(change => change.Sequence).ToArray())); + const int threads = 8; + const int writes = 1000; + + Parallel.For(0, threads, new ParallelOptions { MaxDegreeOfParallelism = threads }, thread => + { + for (var i = 0; i < writes; i++) + cache.Commit(batch => + { + batch.Upsert(typeof(JournalNote), new JournalNote { Name = $"{thread}-{i}-a" }, KeyOf($"{thread}-{i}-a")); + batch.Upsert(typeof(JournalNote), new JournalNote { Name = $"{thread}-{i}-b" }, KeyOf($"{thread}-{i}-b")); + }); + }); + + Assert.That(calls, Has.Count.EqualTo(threads * writes), "one call per admission"); + var expected = 1L; + foreach (var call in calls) + { + Assert.That(call, Is.EqualTo(new[] { expected, expected + 1 }), "contiguous, and one call's changes together"); + expected += 2; + } + } + + [Test] + public void AJournalRunsAfterTheAdmissionIsAppliedAndBeforeAnyObserver() + { + using var cache = Memory(); + var order = new List(); + using var observer = cache.Watch().Subscribe(change => order.Add($"observer:{change.Sequence}")); + var seenInside = new List(); + using var journal = cache.AddJournal(changes => + { + order.Add($"journal:{changes[0].Sequence}"); + seenInside.Add(cache.Get(KeyOf("one"))?.Name); + Assert.That(changes[0].Kind, Is.EqualTo(order.Count == 1 ? CultCacheDocumentChangeKind.Added : CultCacheDocumentChangeKind.Updated)); + Assert.That(((JournalNote?)changes[0].Document)?.Power, Is.EqualTo(order.Count == 1 ? 1 : 2)); + Assert.That(((JournalNote?)changes[0].PreviousDocument)?.Power, Is.EqualTo(order.Count == 1 ? null : 1)); + }); + + Put(cache, "one", 1); + Put(cache, "one", 2); + + Assert.That(order, Is.EqualTo(new[] { "journal:1", "observer:1", "journal:2", "observer:2" })); + Assert.That(seenInside, Is.EqualTo(new[] { "one", "one" }), "the journal reads the new state"); + } + + [Test] + public void AJournalIsCalledForAnAdmissionThatChangedSomethingAndNeverForOneThatDidNot() + { + using var cache = Memory(); + var calls = 0; + using var journal = cache.AddJournal(changes => + { + calls++; + Assert.That(changes, Is.Not.Empty); + }); + + Assert.That(cache.Remove(KeyOf("absent")), Is.False); + + Assert.That(calls, Is.Zero); + } + + [Test] + public void JournalsRunInRegistrationOrderAndADisposedJournalStopsRunning() + { + using var cache = Memory(); + var order = new List(); + using var first = cache.AddJournal(_ => order.Add("first")); + var second = cache.AddJournal(_ => order.Add("second")); + using var third = cache.AddJournal(_ => order.Add("third")); + Put(cache, "one"); + + second.Dispose(); + order.Add("|"); + Put(cache, "two"); + + Assert.That(order, Is.EqualTo(new[] { "first", "second", "third", "|", "first", "third" })); + } + + [Test] + public void AJournalThatWritesTheCacheThrowsAndTheAdmissionStands() + { + using var cache = Memory(); + var seen = new List(); + using var observer = cache.Watch().Subscribe(change => seen.Add(change.Sequence)); + using var journal = cache.AddJournal(_ => Put(cache, "nested")); + + var thrown = Assert.Throws(() => Put(cache, "outer"))!; + + Assert.Multiple(() => + { + Assert.That(thrown.Message, Does.Contain("journal")); + Assert.That(cache.Get(KeyOf("outer")), Is.Not.Null, "the admission stands"); + Assert.That(cache.Get(KeyOf("nested")), Is.Null, "the journal's own write did not land"); + Assert.That(seen, Is.EqualTo(new[] { 1L }), "and it was published"); + }); + } + + [Test] + public void AJournalThatRegistersAnotherJournalThrows() + { + using var cache = Memory(); + using var journal = cache.AddJournal(_ => cache.AddJournal(_ => { })); + + var thrown = Assert.Throws(() => Put(cache, "outer"))!; + + Assert.That(thrown.Message, Does.Contain("journal")); + Assert.That(cache.Get(KeyOf("outer")), Is.Not.Null); + } + + [Test] + public void TheCacheIsWritableAgainOnceAJournalHasReturned() + { + using var cache = Memory(); + using var journal = cache.AddJournal(_ => { }); + + Put(cache, "one"); + cache.Commit(batch => batch.Upsert(typeof(JournalNote), new JournalNote { Name = "two" }, KeyOf("two"))); + using var late = cache.AddJournal(_ => { }); + + Assert.That(cache.Get(KeyOf("two")), Is.Not.Null); + } + + [Test] + public void ARefusedAdmissionIsNotJournaled() + { + using var cache = Memory(); + var calls = 0; + using var journal = cache.AddJournal(_ => calls++); + JournalNote.Armed = true; + + Assert.Throws(() => Put(cache, "boom")); + + JournalNote.Armed = false; + Assert.That(calls, Is.Zero); + } + + [Test] + public void AnAdmissionTheStoreDeclinesIsNotJournaled() + { + using var cache = Memory(); + Put(cache, "seed"); + var calls = 0; + using var journal = cache.AddJournal(_ => calls++); + + var outcome = cache.TryCommit(batch => + { + batch.Expect(KeyOf("seed"), null); + batch.Upsert(typeof(JournalNote), new JournalNote { Name = "two" }, KeyOf("two")); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Mismatch)); + Assert.That(calls, Is.Zero); + Assert.That(cache.Get(KeyOf("two")), Is.Null); + } + + [Test] + public void AVariantDependentIsJournaledInTheSameCallAsItsBaseEdit() + { + using var cache = Memory(); + Put(cache, "base", 10); + cache.Commit(batch => batch.UpsertVariant(KeyOf("big"), KeyOf("base"), new[] + { + cache.Override(nameof(JournalNote.Name), "big"), + })); + var calls = new List(); + using var journal = cache.AddJournal(changes => calls.Add(changes.Select(change => change.Key.Value).ToArray())); + + Put(cache, "base", 20); + + Assert.That(calls, Has.Count.EqualTo(1)); + Assert.That(calls[0], Is.EquivalentTo(new[] { KeyOf("base").Value, KeyOf("big").Value })); + } + + [Test] + public void AThrowingJournalDoesNotStopTheOthersOrTheAdmissionAndTheWriterHearsOfItAfterPublication() + { + using var cache = Memory(); + var order = new List(); + using var observer = cache.Watch().Subscribe(_ => order.Add("observer")); + using var a = cache.AddJournal(_ => + { + order.Add("a"); + throw new InvalidOperationException("a"); + }); + using var b = cache.AddJournal(_ => order.Add("b")); + using var c = cache.AddJournal(_ => + { + order.Add("c"); + throw new InvalidOperationException("c"); + }); + + var thrown = Assert.Throws(() => Put(cache, "one"))!; + + Assert.Multiple(() => + { + Assert.That(thrown.InnerExceptions.Select(exception => exception.Message), Is.EqualTo(new[] { "a", "c" })); + Assert.That(order, Is.EqualTo(new[] { "a", "b", "c", "observer" }), "every journal ran, then the change published"); + Assert.That(cache.Get(KeyOf("one")), Is.Not.Null, "the admission stands"); + }); + + using var quiet = cache.Watch().Subscribe(_ => { }); + Assert.Throws(() => Put(cache, "two"), "a throwing journal stays registered"); + } + + [Test] + public void AJournalThatThrowsDuringALoadLeavesTheStoreAttachedAndTheLoadAdopted() + { + var path = Path.Combine(_directory, "notes.cc"); + using (var writer = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry })) + Put(writer, "stored"); + + var cache = Memory(); + var seen = new List(); + using var observer = cache.Watch().Subscribe(change => seen.Add(change.Sequence)); + using var journal = cache.AddJournal(_ => throw new InvalidOperationException("journal boom")); + + var thrown = Assert.Throws(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)))!; + + Assert.Multiple(() => + { + Assert.That(thrown.Message, Is.EqualTo("journal boom")); + Assert.That(cache.BackingStores, Has.Count.EqualTo(1), "the store stays attached: it and the cache agree"); + Assert.That(cache.Get(KeyOf("stored")), Is.Not.Null); + Assert.That(seen, Is.EqualTo(new[] { 1L })); + }); + } + } +} From b579075dd52597093f9bc23ee07c37976bf2392d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:41:56 +0200 Subject: [PATCH 40/64] GameCult.Networking: a declared length past int.MaxValue is malformed; server mode is claimed only by a Connect; a refusing session ends CultNetRudpPacketCodec.Decode did checked((int)length), so a datagram declaring 0x80000000 threw OverflowException, which no receive loop catches; the length is now compared as a long and takes the ordinary malformed-packet drop. Server mode claimed its endpoint on the first datagram of any type; only a Connect claims it now. A session that refuses a packet has already recorded its reliable sequence, so the socket transport and the listener end it: the session forgets the peer (so the goodbye does not acknowledge the refused frame), the peer gets a Disconnect, the listener drops the peer and raises PeerDisconnected, and a server-mode transport forgets its endpoint. CultNetRudpSession gains ResetPeerState, and the session and the listener reject an InitialSequence of uint.MaxValue. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 130 +++++++++-- .../NetworkingTests.cs | 212 ++++++++++++++++++ 2 files changed, 328 insertions(+), 14 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 9df650cd..b354d757 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -664,12 +664,15 @@ namespace GameCult.Networking throw new InvalidOperationException("CultNet RUDP packet header length does not match the channel id length."); } - var payloadLength = checked((int)BinaryPrimitives.ReadUInt32BigEndian(wire.AsSpan(30, 4))); - if (wire.Length != headerBytes + payloadLength) + // Compared as a long: a declared length past int.MaxValue is a malformed packet, not an overflow. + var declaredPayloadLength = BinaryPrimitives.ReadUInt32BigEndian(wire.AsSpan(30, 4)); + if (declaredPayloadLength != (long)wire.Length - headerBytes) { throw new InvalidOperationException("CultNet RUDP packet payload length does not match the packet size."); } + var payloadLength = (int)declaredPayloadLength; + var flags = wire[6]; var payload = new byte[payloadLength]; Array.Copy(wire, headerBytes, payload, 0, payload.Length); @@ -923,10 +926,7 @@ namespace GameCult.Networking public CultNetRudpSession(CultNetRudpSessionOptions options) { if (options == null) throw new ArgumentNullException(nameof(options)); - if (options.MaxPendingReliablePackets.HasValue && options.MaxPendingReliablePackets.Value <= 0) - { - throw new ArgumentOutOfRangeException(nameof(options), "RUDP MaxPendingReliablePackets must be greater than zero."); - } + ValidateLimits(options.InitialSequence, options.MaxPendingReliablePackets); ConnectionId = options.ConnectionId; _nextSequence = options.InitialSequence; @@ -934,6 +934,25 @@ namespace GameCult.Networking _maxPendingReliablePackets = options.MaxPendingReliablePackets; } + internal static readonly byte[] RefusedPacketReason = Encoding.UTF8.GetBytes("session refused a packet"); + + /// + /// A session cannot admit a Connect when its sequence space starts exhausted or its reliable + /// queue holds nothing, so neither is a usable configuration. + /// + internal static void ValidateLimits(uint initialSequence, int? maxPendingReliablePackets) + { + if (initialSequence == uint.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(initialSequence), "RUDP InitialSequence must leave room for a reliable packet."); + } + + if (maxPendingReliablePackets.HasValue && maxPendingReliablePackets.Value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxPendingReliablePackets), "RUDP MaxPendingReliablePackets must be greater than zero."); + } + } + /// /// Gets the connection/session binding id. /// @@ -1237,6 +1256,38 @@ namespace GameCult.Networking return CreatePacket(CultNetRudpPacketType.Ping, "control", payload ?? Array.Empty(), reliable: false, ordered: false, sequenced: false); } + /// + /// Forgets everything learned from the peer; sequence numbers already issued stay issued. + /// + public void ResetPeerState() + { + _connected = false; + _lastReceivedAtMs = null; + _highestReceivedSequence = null; + _receivedSequences.Clear(); + _nextSequencedByChannel.Clear(); + _latestSequencedByChannel.Clear(); + lock (_pendingReliableGate) + { + _pendingReliable.Clear(); + _queuedReliable.Clear(); + } + _orderedNextSequenceByChannel.Clear(); + _orderedBuffers.Clear(); + _fragmentBuffers.Clear(); + } + + /// + /// Ends a session that refused a packet. Receive has already recorded the packet's reliable + /// sequence, so the peer's sequences are forgotten before the goodbye is built: its ack field + /// would otherwise acknowledge the very frame the session refused. + /// + internal CultNetRudpPacket EndRefused() + { + ResetPeerState(); + return CreateDisconnect(RefusedPacketReason); + } + /// /// Creates a packet carrying a transport-level disconnect reason. /// @@ -2112,6 +2163,13 @@ namespace GameCult.Networking } if (_remoteEndPoint == null) { + // Only a Connect claims the endpoint of a server-mode transport. + if (_mode == CultNetRudpSocketMode.Server && packet.PacketType != CultNetRudpPacketType.Connect) + { + _stats.PacketsDropped++; + return true; + } + _remoteEndPoint = remote; } else if (!_remoteEndPoint.Equals(remote)) @@ -2150,7 +2208,24 @@ namespace GameCult.Networking } catch (InvalidOperationException) { + // Receive has already recorded the packet's reliable sequence, so the session + // cannot be kept: a retransmit would be acknowledged and the frame silently lost. + // End it and tell the peer; a server-mode transport also forgets its endpoint, so + // only a Connect can claim it again. _stats.PacketsDropped++; + var goodbye = _session.EndRefused(); + DisconnectReason = CultNetRudpSession.RefusedPacketReason; + try + { + SendPacket(goodbye); + } + catch (SocketException) + { + // Best effort: the session ends whether or not the peer hears it. + } + + if (_mode == CultNetRudpSocketMode.Server) + _remoteEndPoint = null; return true; } if (result.Reply != null) @@ -2388,6 +2463,7 @@ namespace GameCult.Networking _socket = options.Socket ?? throw new ArgumentNullException(nameof(options.Socket)); if (_socket.ReceiveBufferSize < MinimumReceiveBufferBytes) _socket.ReceiveBufferSize = MinimumReceiveBufferBytes; + CultNetRudpSession.ValidateLimits(options.InitialSequence, options.MaxPendingReliablePackets); _connectionId = options.ConnectionId; _initialSequence = options.InitialSequence; _resendDelayMs = options.ResendDelayMs; @@ -2561,25 +2637,51 @@ namespace GameCult.Networking return true; } - CultNetRudpReceiveResult result; + CultNetRudpReceiveResult? received; CultNetRudpPacket? acknowledgement = null; lock (existingPeer.SessionGate) { try { - result = existingPeer.Session.Receive(packet, NowMs()); + received = existingPeer.Session.Receive(packet, NowMs()); } catch (InvalidOperationException) { + // Receive has already recorded the packet's reliable sequence, so the session + // cannot be kept: a retransmit would be acknowledged and the frame silently lost. _stats.PacketsDropped++; - return true; + received = null; + } + + if (received == null) + { + try + { + SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); + } + catch (SocketException) + { + // Best effort: the session ends whether or not the peer hears it. + } + } + else + { + if (received.Reply != null) + SendPacket(existingPeer.RemoteEndPoint, received.Reply); + SendPackets(existingPeer, received.ReadyToSend); + if (packet.PacketType == CultNetRudpPacketType.Data) + acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } - if (result.Reply != null) - SendPacket(existingPeer.RemoteEndPoint, result.Reply); - SendPackets(existingPeer, result.ReadyToSend); - if (packet.PacketType == CultNetRudpPacketType.Data) - acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } + if (received == null) + { + existingPeer.DisconnectReason = CultNetRudpSession.RefusedPacketReason; + _peers.Remove(peerKey); + PeerDisconnected?.Invoke(existingPeer); + return true; + } + + var result = received; if (result.Disconnected) { existingPeer.DisconnectReason = result.DisconnectReason; diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index d94b877e..94b3dd4a 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1255,6 +1255,218 @@ namespace GameCult.Networking.Tests Assert.That(Encoding.UTF8.GetString(ReceiveRudpFrame(client).Payload), Is.EqualTo("server still here")); } + private static List DrainPackets(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + /// A reliable frame no session accepts: a fragment set with no fragment id. + private static CultNetRudpPacket PoisonedFrame(CultNetRudpSession session) + { + var packet = session.Send("schema", Encoding.UTF8.GetBytes("poison"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + packet.FragmentCount = 2; + packet.FragmentId = 0; + return packet; + } + + [Test] + public void RudpPacketCodec_TreatsADeclaredLengthPastInt32AsMalformed() + { + var wire = CultNetRudpPacketCodec.Encode(new CultNetRudpPacket + { + PacketType = CultNetRudpPacketType.Ack, + ConnectionId = 0x10203054, + ChannelId = "c" + }); + Assert.That(wire, Has.Length.EqualTo(36)); + wire[30] = 0x80; + wire[31] = 0; + wire[32] = 0; + wire[33] = 0; + Assert.Throws(() => CultNetRudpPacketCodec.Decode(wire)); + + using var connectionSocket = BindUdpSocket(); + using var listenerSocket = BindUdpSocket(); + using var senderSocket = BindUdpSocket(); + using var connection = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-connection", + Socket = connectionSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = 0x10203054 + }); + using var listener = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = listenerSocket, + ConnectionId = 0x10203054 + }); + senderSocket.SendTo(wire, connectionSocket.LocalEndPoint!); + senderSocket.SendTo(wire, listenerSocket.LocalEndPoint!); + Assert.That(connection.ReceiveOnce(), Is.Null); + Assert.That(listener.ReceiveOnce(), Is.Null); + Assert.That(connection.Stats.PacketsDropped, Is.EqualTo(1)); + Assert.That(listener.Stats.PacketsDropped, Is.EqualTo(1)); + } + + [Test] + public void RudpServerMode_IsClaimedOnlyByAConnect() + { + using var serverSocket = BindUdpSocket(); + using var orphanSocket = BindUdpSocket(); + using var clientSocket = BindUdpSocket(); + const uint connectionId = 0x10203055; + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = connectionId + }); + using var client = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = clientSocket, + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = serverSocket.LocalEndPoint!, + ConnectionId = connectionId + }); + var orphan = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }); + + // A right-id packet that is not a Connect must claim nothing. + orphanSocket.SendTo(CultNetRudpPacketCodec.Encode(orphan.CreateAck()), serverSocket.LocalEndPoint!); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + + client.Connect(Encoding.UTF8.GetBytes("join")); + PumpRudpHandshake(client, server); + Assert.That(client.Connected, Is.True); + Assert.That(server.Connected, Is.True); + } + + [Test] + public void RudpSocketTransport_EndsTheSessionThatRefusesAReliableFrame() + { + using var serverSocket = BindUdpSocket(); + using var peerSocket = BindUdpSocket(); + var serverEndPoint = serverSocket.LocalEndPoint!; + const uint connectionId = 0x10203056; + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = connectionId + }); + var peer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }); + + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(peer.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Receive(DrainPackets(peerSocket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + + var poison = PoisonedFrame(peer); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + Assert.That(server.Connected, Is.False); + Assert.That(server.DisconnectReason, Is.Not.Null); + + // The peer is told, and the goodbye does not acknowledge the refused frame. + var goodbye = DrainPackets(peerSocket).Single(); + Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + + // A retransmit is dropped, not acknowledged. + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(2)); + Assert.That(DrainPackets(peerSocket), Is.Empty); + + // A Connect claims the endpoint again. + var again = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(again.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + Assert.That(DrainPackets(peerSocket).Select(p => p.PacketType), Is.EqualTo(new[] { CultNetRudpPacketType.Accept })); + } + + [Test] + public void RudpSocketTransportServer_EndsThePeerWhoseReliableFrameItRefuses() + { + using var serverSocket = BindUdpSocket(); + using var peerSocket = BindUdpSocket(); + var serverEndPoint = serverSocket.LocalEndPoint!; + const uint connectionId = 0x10203057; + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = connectionId + }); + var ended = new List(); + server.PeerDisconnected += ended.Add; + var peer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }); + + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(peer.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Receive(DrainPackets(peerSocket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + + var poison = PoisonedFrame(peer); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + Assert.That(server.Peers, Is.Empty); + Assert.That(ended, Has.Count.EqualTo(1)); + + var goodbye = DrainPackets(peerSocket).Single(); + Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(2)); + Assert.That(DrainPackets(peerSocket), Is.Empty); + } + + [Test] + public void RudpConfiguration_RejectsLimitsThatCannotAdmitAConnect() + { + Assert.Throws(() => + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 1, InitialSequence = uint.MaxValue })); + Assert.Throws(() => + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 1, MaxPendingReliablePackets = 0 })); + Assert.DoesNotThrow(() => + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 1, InitialSequence = uint.MaxValue - 1, MaxPendingReliablePackets = 1 })); + + using var socket = BindUdpSocket(); + Assert.Throws(() => + new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = socket, + ConnectionId = 1, + InitialSequence = uint.MaxValue + })); + Assert.Throws(() => + new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = socket, + ConnectionId = 1, + MaxPendingReliablePackets = 0 + })); + } + [Test] public void RudpSocketTransport_CarriesCultNetSchemaMessages() { From 9dba01082c53934e608eb2978e549deb0230bb0e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:42:45 +0200 Subject: [PATCH 41/64] GameCult.Networking: the listener refusal local no longer shadows the datagram length Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index b354d757..03543ffe 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -2637,23 +2637,23 @@ namespace GameCult.Networking return true; } - CultNetRudpReceiveResult? received; + CultNetRudpReceiveResult? outcome; CultNetRudpPacket? acknowledgement = null; lock (existingPeer.SessionGate) { try { - received = existingPeer.Session.Receive(packet, NowMs()); + outcome = existingPeer.Session.Receive(packet, NowMs()); } catch (InvalidOperationException) { // Receive has already recorded the packet's reliable sequence, so the session // cannot be kept: a retransmit would be acknowledged and the frame silently lost. _stats.PacketsDropped++; - received = null; + outcome = null; } - if (received == null) + if (outcome == null) { try { @@ -2666,14 +2666,14 @@ namespace GameCult.Networking } else { - if (received.Reply != null) - SendPacket(existingPeer.RemoteEndPoint, received.Reply); - SendPackets(existingPeer, received.ReadyToSend); + if (outcome.Reply != null) + SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); + SendPackets(existingPeer, outcome.ReadyToSend); if (packet.PacketType == CultNetRudpPacketType.Data) acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } } - if (received == null) + if (outcome == null) { existingPeer.DisconnectReason = CultNetRudpSession.RefusedPacketReason; _peers.Remove(peerKey); @@ -2681,7 +2681,7 @@ namespace GameCult.Networking return true; } - var result = received; + var result = outcome; if (result.Disconnected) { existingPeer.DisconnectReason = result.DisconnectReason; From 306412e822b9c29a4968efeb2df14500d88af81b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:43:55 +0200 Subject: [PATCH 42/64] GameCult.Networking.Tests: the malformed-length datagram has the 36-byte fixed header and no channel id Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Networking.Tests/NetworkingTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 94b3dd4a..ef4d5a3d 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1284,7 +1284,7 @@ namespace GameCult.Networking.Tests { PacketType = CultNetRudpPacketType.Ack, ConnectionId = 0x10203054, - ChannelId = "c" + ChannelId = string.Empty }); Assert.That(wire, Has.Length.EqualTo(36)); wire[30] = 0x80; From 97405518398c2736222ec1187f7e8e8968ace58a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:45:38 +0200 Subject: [PATCH 43/64] cultnet-ts: pin the operation service handler rejection path as a counted drop The fragment-id-0 test is caught by the session-refusal branch, so a frame that fails inside the handler needs its own test. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/test/cultnet.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index b9aaa2a5..9761b37f 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2028,6 +2028,17 @@ test("operation service drops and counts a packet its session refuses and keeps await waitFor(() => server.packetsDropped === 1, "the poisoned frame dropped"); await waitFor(() => received.some((p) => p.packetType === "disconnect"), "the session ended"); + // A frame the session accepts but the service cannot handle rejects inside + // the handler; that too is a dropped packet, not a dead process. + const garbage = new CultNetRudpSession({ connectionId }); + received.length = 0; + toServer(garbage.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the second Accept"); + garbage.receive(received[0]!, 0); + const [notMessagePack] = garbage.sendMany("schema", Buffer.from([0xc1]), { reliable: true, ordered: true, nowMs: 0 }); + toServer(notMessagePack!); + await waitFor(() => server.packetsDropped === 2, "the unhandled frame dropped"); + const response = await invokeCultNetOperation(server.endpoint, { schemaVersion: "cultnet.operation_request.v0", messageId: "after-poison", From 5b3885a1e8c48cd489bf46cf42d51b2c3e8ce660 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:47:38 +0200 Subject: [PATCH 44/64] Detect an unsubscribed observer by what the cache still holds, and pin the empty admission A disposed R3 subscription ignores OnNext, so a no-op removal from the cache's observer array cannot be seen by what a receiver gets; the test now checks the cache stops holding the subscriber. An admission that changed nothing (a store attached with no records) is not journaled. Co-Authored-By: Claude Opus 5.5 --- .../CultCacheJournalTests.cs | 8 ++--- .../CultCacheObserverTests.cs | 30 +++++++++++++++---- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs index 563669e0..287af93b 100644 --- a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs @@ -124,12 +124,10 @@ namespace GameCult.Caching.Tests { using var cache = Memory(); var calls = 0; - using var journal = cache.AddJournal(changes => - { - calls++; - Assert.That(changes, Is.Not.Empty); - }); + using var journal = cache.AddJournal(_ => calls++); + // Attaching a store with nothing in it is an admission with no change. + cache.AddBackingStore(new SingleFileMessagePackBackingStore(Path.Combine(_directory, "empty.cc"))); Assert.That(cache.Remove(KeyOf("absent")), Is.False); Assert.That(calls, Is.Zero); diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs index fa87ea4f..b26d4ca4 100644 --- a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using GameCult.Caching; using GameCult.Caching.MessagePack; using MessagePack; @@ -123,17 +124,36 @@ namespace GameCult.Caching.Tests } [Test] - public void AnUnsubscribedObserverStopsReceiving() + public void AnUnsubscribedObserverIsReleasedByTheCache() { using var cache = new CultCache(Registry); - var seen = new List(); - var subscription = cache.Watch().Subscribe(change => seen.Add(change.Document!.Text)); + var target = Subscribe(cache, out var subscription); Send(cache, "one"); + CollectGarbage(); + Assert.That(target.IsAlive, Is.True, "a live subscription is held by the cache"); subscription.Dispose(); - Send(cache, "two"); + CollectGarbage(); - Assert.That(seen, Is.EqualTo(new[] { "one" })); + Assert.That(target.IsAlive, Is.False, "an unsubscribed observer is no longer held"); + GC.KeepAlive(cache); + } + + // A disposed R3 subscription ignores OnNext, so a stale entry in the cache is invisible to a receiver; it shows as + // a leak: the cache keeps the subscriber alive. + [MethodImpl(MethodImplOptions.NoInlining)] + private static WeakReference Subscribe(CultCache cache, out IDisposable subscription) + { + var target = new object(); + subscription = cache.Watch().Subscribe(_ => GC.KeepAlive(target)); + return new WeakReference(target); + } + + private static void CollectGarbage() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); } // A store that admits a load and then fails: the hold's body throws after the changes were minted. From 281e1b238c2383f85ece604e2581a65dc5b14c00 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:03:08 +0200 Subject: [PATCH 45/64] Keep the test frame from holding the subscription it checks the cache released Co-Authored-By: Claude Opus 5.5 --- .../CultCacheObserverTests.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs index b26d4ca4..b2c8e7f5 100644 --- a/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheObserverTests.cs @@ -127,25 +127,32 @@ namespace GameCult.Caching.Tests public void AnUnsubscribedObserverIsReleasedByTheCache() { using var cache = new CultCache(Registry); - var target = Subscribe(cache, out var subscription); + var kept = new List(); + var live = Subscribe(cache, kept, dispose: false); + var gone = Subscribe(cache, kept, dispose: true); Send(cache, "one"); CollectGarbage(); - Assert.That(target.IsAlive, Is.True, "a live subscription is held by the cache"); - subscription.Dispose(); - CollectGarbage(); - - Assert.That(target.IsAlive, Is.False, "an unsubscribed observer is no longer held"); + Assert.Multiple(() => + { + Assert.That(live.IsAlive, Is.True, "a live subscription is held by the cache"); + Assert.That(gone.IsAlive, Is.False, "an unsubscribed observer is no longer held"); + }); + GC.KeepAlive(kept); GC.KeepAlive(cache); } // A disposed R3 subscription ignores OnNext, so a stale entry in the cache is invisible to a receiver; it shows as - // a leak: the cache keeps the subscriber alive. + // a leak: the cache keeps the subscriber alive. The test frame must not hold the subscription itself. [MethodImpl(MethodImplOptions.NoInlining)] - private static WeakReference Subscribe(CultCache cache, out IDisposable subscription) + private static WeakReference Subscribe(CultCache cache, List kept, bool dispose) { var target = new object(); - subscription = cache.Watch().Subscribe(_ => GC.KeepAlive(target)); + var subscription = cache.Watch().Subscribe(_ => GC.KeepAlive(target)); + if (dispose) + subscription.Dispose(); + else + kept.Add(subscription); return new WeakReference(target); } From 36acb4295ad0f079663c343bd94f45a969692c26 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:04:59 +0200 Subject: [PATCH 46/64] Reach the empty admission through a commit that removes nothing Attaching a store with no file never calls Loaded, so it did not exercise the rule; a commit removing a key the cache does not hold admits nothing. Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/CultCacheJournalTests.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs index 287af93b..96e04d5f 100644 --- a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs @@ -120,15 +120,14 @@ namespace GameCult.Caching.Tests } [Test] - public void AJournalIsCalledForAnAdmissionThatChangedSomethingAndNeverForOneThatDidNot() + public void AnAdmissionThatChangesNothingIsNotJournaled() { using var cache = Memory(); var calls = 0; using var journal = cache.AddJournal(_ => calls++); - // Attaching a store with nothing in it is an admission with no change. - cache.AddBackingStore(new SingleFileMessagePackBackingStore(Path.Combine(_directory, "empty.cc"))); - Assert.That(cache.Remove(KeyOf("absent")), Is.False); + // Removing a record the cache does not hold is a commit that admits nothing. + cache.Commit(batch => batch.Remove(KeyOf("absent"))); Assert.That(calls, Is.Zero); } From 592a2d49d4122183bec0c525922002a13216cbce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:35:50 +0200 Subject: [PATCH 47/64] Journal registration counts as disposed only once removal succeeds; a throwing hold body surfaces journal failures AddJournal's Disposable.Create counted a refused dispose (from inside a journal or another cache's hold) as done, leaving the journal registered forever. A private registration now flips only after the removal succeeded. Held's catch path dropped the hold's journal failures; it now throws an AggregateException with the body's exception first. The store-composition contract states exactly when the second getter read runs on a conditional commit. Adds the direct test that a load is journaled. Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-store-composition.md | 10 +- src/GameCult.Caching/CultCache.cs | 31 +++++- .../CultCacheJournalTests.cs | 101 ++++++++++++++++++ 3 files changed, 135 insertions(+), 7 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 251a1c1e..a70814d5 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -101,10 +101,12 @@ which route wrote it. already agree when publication starts. - An admission runs a record's `[CultName]` and `[CultIndex]` getters once per landing record, before the store is written and before any `Sequence` is - minted. A conditional commit on a store that holds variants is judged a second - time against the file it merges onto, which runs the getters again for the - records it lands and for any variant it re-resolves; that judgement is also - before the store is written. A getter that throws refuses the whole change + minted. A conditional commit on a clean file store is judged a second time + against the file it merges onto whenever the cache holds variants in any store, + the batch lands a variant, or the file holds one. That second read runs the + getters again for the records the batch lands, for any variant it re-resolves, + and for the records other writers put in the file (the arriving set); that + judgement is also before the store is written. A getter that throws refuses the whole change set, loads included: nothing is written, minted or left in memory. - Pulling all stores pulls every attached store even if a handler throws during one store's load, then rethrows. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 986ea974..e8ec8fd6 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1903,6 +1903,29 @@ namespace GameCult.Caching change.Sequence)); } + // Disposing removes the journal. A dispose the cache refuses (from inside a journal, or inside another cache's hold) + // throws and leaves the registration live and disposable; it counts as disposed only once the removal succeeded. + private sealed class JournalRegistration : IDisposable + { + private readonly CultCache _cache; + private readonly Action>> _journal; + private bool _removed; + + public JournalRegistration(CultCache cache, Action>> journal) + { + _cache = cache; + _journal = journal; + } + + public void Dispose() + { + if (_removed) + return; + _cache.Held(() => _cache._journals.Remove(_journal)); + _removed = true; + } + } + private IDisposable Register(Observer observer) { lock (_observerGate) @@ -1941,7 +1964,7 @@ namespace GameCult.Caching _journals.Add(journal); return true; }); - return Disposable.Create(() => Held(() => _journals.Remove(journal))); + return new JournalRegistration(this, journal); } // Attaching reads the store; there is no interval in which it is attached but unread. @@ -2425,11 +2448,13 @@ namespace GameCult.Caching lock (_gate) result = body(); } - catch + catch (Exception exception) { _held = null; Publish(mine); - throw; + if (mine.Failures.Count == 0) + throw; + throw new AggregateException(mine.Failures.Prepend(exception)); } _held = null; diff --git a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs index 96e04d5f..404a75cb 100644 --- a/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs +++ b/tests/GameCult.Caching.Tests/CultCacheJournalTests.cs @@ -275,6 +275,107 @@ namespace GameCult.Caching.Tests Assert.Throws(() => Put(cache, "two"), "a throwing journal stays registered"); } + [Test] + public void ADisposeTheCacheRefusesLeavesTheJournalRegisteredAndALaterDisposeRemovesIt() + { + using var cache = Memory(); + var calls = 0; + IDisposable? registration = null; + registration = cache.AddJournal(_ => + { + calls++; + registration!.Dispose(); + }); + + var thrown = Assert.Throws(() => Put(cache, "one"))!; + Assert.That(thrown.Message, Does.Contain("journal"), "disposing from inside a journal is refused"); + + registration.Dispose(); + Put(cache, "two"); + + Assert.That(calls, Is.EqualTo(1), "the outside dispose removed it, though the first attempt threw"); + registration.Dispose(); + } + + [Test] + public void ALoadIsJournaledWithItsChanges() + { + var path = Path.Combine(_directory, "notes.cc"); + using (var writer = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry })) + { + Put(writer, "a"); + Put(writer, "b"); + } + + using var cache = Memory(); + var journaled = new List>>(); + using var journal = cache.AddJournal(changes => journaled.Add(changes)); + + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + + Assert.Multiple(() => + { + Assert.That(journaled, Has.Count.EqualTo(1), "one admission, one call"); + Assert.That(journaled[0].Select(change => change.Key).OrderBy(key => key.Value), + Is.EqualTo(new[] { KeyOf("a"), KeyOf("b") })); + }); + } + + // A store that admits a load and then fails: the hold's body throws after the journal has already failed. + private sealed class LoadThenFailStore : CacheBackingStore + { + public bool Armed; + + public override void PullAll() => Held(() => + { + if (!Armed) + return; + var record = new CultStoredDocument( + KeyOf("loaded"), "2026-01-01T00:00:00.0000000+00:00", Registry.GetRequired(typeof(JournalNote)), new JournalNote { Name = "loaded" }); + Loaded!(new[] { record }, Array.Empty()); + throw new InvalidOperationException("store failed after the load"); + }); + + public override void Push(CultStoredDocument entry) { } + public override void Delete(CultStoredDocument entry) { } + public override void PushAll() { } + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => CultCommitOutcome.Committed; + } + + [Test] + public void AHoldWhoseBodyThrowsSurfacesTheBodyFirstThenTheJournalFailures() + { + using var cache = Memory(); + var store = new LoadThenFailStore(); + cache.AddBackingStore(store); + using var a = cache.AddJournal(_ => throw new InvalidOperationException("journal a")); + using var b = cache.AddJournal(_ => throw new InvalidOperationException("journal b")); + store.Armed = true; + + var thrown = Assert.Throws(() => store.PullAll())!; + + Assert.Multiple(() => + { + Assert.That(thrown.InnerExceptions.Select(exception => exception.Message), + Is.EqualTo(new[] { "store failed after the load", "journal a", "journal b" })); + Assert.That(cache.Get(KeyOf("loaded")), Is.Not.Null, "the load landed"); + }); + } + + [Test] + public void AHoldWhoseBodyThrowsAloneRethrowsItAsItself() + { + using var cache = Memory(); + var store = new LoadThenFailStore(); + cache.AddBackingStore(store); + using var journal = cache.AddJournal(_ => { }); + store.Armed = true; + + var thrown = Assert.Throws(() => store.PullAll())!; + + Assert.That(thrown.Message, Is.EqualTo("store failed after the load")); + } + [Test] public void AJournalThatThrowsDuringALoadLeavesTheStoreAttachedAndTheLoadAdopted() { From c7a61018da87a5f05b91ecb31ddb0e1dc6e4c366 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:36:58 +0200 Subject: [PATCH 48/64] cultnet-rs, cultmesh-rs: an ended session fails its flush; issued sequences survive a reset; the doc server says goodbye A session that refused a packet, or was disconnected, forgot its unacknowledged writes, so flush_reliable returned success for writes the peer never acknowledged. The transport now fails the flush with the reason and reads such writes as Invalidated. Reset no longer reissues sequence numbers or fragment ids: a late duplicate Connect in server mode reused sequences the peer had seen. end_refused_session owns the reset-then-goodbye ordering for the hub, the transport and the document server, which used to end a refusing session silently. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 14 ++- .../cultmesh-rs/tests/rudp_document_server.rs | 9 ++ packages/cultnet-rs/src/rudp.rs | 60 +++++---- .../cultnet-rs/tests/rudp_stray_packets.rs | 115 ++++++++++++++++++ 4 files changed, 175 insertions(+), 23 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 5536add0..b99c7145 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -325,7 +325,7 @@ where let result = match result { Ok(result) => result, Err(_) => { - self.sessions.remove(&key); + self.end_refused_session(key)?; self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } @@ -418,7 +418,7 @@ where match entry.session.answer_repeated_connect(packet, now) { Ok(reply) => reply, Err(_) => { - self.sessions.remove(&key); + self.end_refused_session(key)?; return Ok(false); } } @@ -592,6 +592,16 @@ where Ok(None) } + /// The session refused a packet: it ends, and the client is told so it does + /// not wait on a session the server no longer holds. + fn end_refused_session(&mut self, key: CultMeshRudpSessionKey) -> Result<()> { + let Some(mut entry) = self.sessions.remove(&key) else { + return Ok(()); + }; + let goodbye = entry.session.end_refused_session(); + self.send_packet(key.remote_addr, &goodbye) + } + fn payload_budget_allows(&self, key: CultMeshRudpSessionKey, bytes: usize) -> bool { let Some(session_bytes) = self .sessions diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index d3ee1de8..fa741827 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -627,6 +627,15 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { send_raw(&mut server, &poison)?; assert_eq!(server.packets_dropped(), 3); assert_eq!(server.session_count(), 1); + // ...and the refused client is told, after the Accepts its storm drew. + raw.set_read_timeout(Some(Duration::from_millis(200)))?; + let mut buffer = vec![0_u8; 65_535]; + let mut told = false; + while let Ok(received) = raw.recv(&mut buffer) { + told |= cultnet_rs::decode_rudp_packet(&buffer[..received])?.packet_type + == cultnet_rs::CultNetRudpPacketType::Disconnect; + } + assert!(told, "the refused client must be sent a goodbye"); send( &mut real, diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 00e0e61b..55a8d8cf 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -334,11 +334,12 @@ impl CultNetRudpSession { Ok(()) } + /// Forgets everything learned from the peer. Sequence numbers and fragment + /// ids already issued stay issued: a peer that still remembers them must + /// never see one reused. pub fn reset_peer_state(&mut self) { self.session_scope = Uuid::new_v4(); - self.next_sequence = self.initial_sequence; self.next_sequenced_by_channel.clear(); - self.next_fragment_id = 1; self.connected = false; self.last_received_at_ms = None; self.highest_received_sequence = None; @@ -351,6 +352,16 @@ impl CultNetRudpSession { self.fragment_buffers.clear(); } + /// Ends a session that refused a packet. `receive` has already recorded the + /// refused packet's reliable sequence, so the session cannot be kept: a + /// retransmit would be acknowledged and the frame silently lost. Returns the + /// goodbye for the peer, built after the reset or its ack field would + /// acknowledge the very frame that was refused. + pub fn end_refused_session(&mut self) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) + } + pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { self.ensure_reliable_capacity(1)?; let packet = self.create_packet( @@ -1729,15 +1740,11 @@ impl CultNetRudpServerHub { let Some(mut peer) = self.peers.remove(&remote_addr) else { return Ok(()); }; - let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); - // Forget the peer's sequences first: the goodbye's ack field would - // otherwise acknowledge the very frame the session refused. - peer.session.reset_peer_state(); - let goodbye = peer.session.create_disconnect(reason.clone()); + let goodbye = peer.session.end_refused_session(); self.pending_events .push_back(CultNetRudpServerEvent::Disconnected { session: peer.context, - reason, + reason: goodbye.payload.clone(), }); self.send_packet(remote_addr, &goodbye) } @@ -1861,6 +1868,7 @@ impl CultNetRudpSocketTransportConnection { )); } let packet = self.session.create_connect(now_ms(), payload)?; + self.disconnect_reason = None; self.send_packet(&packet) } @@ -1914,7 +1922,12 @@ impl CultNetRudpSocketTransportConnection { &self, receipt: &CultNetRudpReliableSendReceipt, ) -> CultNetRudpReliableSendStatus { - self.session.reliable_send_status(receipt) + let status = self.session.reliable_send_status(receipt); + // A write still unacknowledged when the session ended never will be. + if status == CultNetRudpReliableSendStatus::Pending && self.disconnect_reason.is_some() { + return CultNetRudpReliableSendStatus::Invalidated; + } + status } fn send_data_packets(&mut self, packets: &[CultNetRudpPacket]) -> Result<()> { @@ -1981,7 +1994,18 @@ impl CultNetRudpSocketTransportConnection { let deadline = Instant::now() + timeout; let mut preserved_frames = VecDeque::new(); let result = (|| { - while self.outstanding_reliable_packet_count() > 0 { + loop { + // An ended session forgot its unacknowledged writes; reporting + // them flushed would be a lie. + if let Some(reason) = &self.disconnect_reason { + return Err(anyhow!( + "RUDP session ended before its reliable writes were acknowledged: {}", + String::from_utf8_lossy(reason) + )); + } + if self.outstanding_reliable_packet_count() == 0 { + return Ok(()); + } if Instant::now() >= deadline { return Err(anyhow!( "RUDP reliable flush timed out with {} packets outstanding", @@ -1993,7 +2017,6 @@ impl CultNetRudpSocketTransportConnection { } self.poll_resends()?; } - Ok(()) })(); preserved_frames.append(&mut self.delivered_frames); @@ -2076,6 +2099,7 @@ impl CultNetRudpSocketTransportConnection { && packet.packet_type == CultNetRudpPacketType::Connect { self.session.reset_peer_state(); + self.disconnect_reason = None; let Ok(accept) = self.session.accept_connect(&packet, now_ms(), Vec::new()) else { self.stats.packets_dropped += 1; return Ok(true); @@ -2085,17 +2109,11 @@ impl CultNetRudpSocketTransportConnection { } let Ok(result) = self.session.receive(&packet, now_ms()) else { - // `receive` has already recorded the packet's reliable sequence, so - // the session cannot be kept: a retransmit would be acknowledged - // and the frame silently lost. End it and tell the peer. + // End the session and tell the peer. A server-mode transport also + // forgets its peer: only a Connect can claim it again. self.stats.packets_dropped += 1; - // The goodbye is built after the reset, or its ack field would - // acknowledge the very frame the session refused. A server-mode - // transport also forgets its peer: only a Connect can claim it again. - let reason = RUDP_REFUSED_PACKET_REASON.to_vec(); - self.session.reset_peer_state(); - let goodbye = self.session.create_disconnect(reason.clone()); - self.disconnect_reason = Some(reason); + let goodbye = self.session.end_refused_session(); + self.disconnect_reason = Some(goodbye.payload.clone()); let sent = self.send_packet(&goodbye); if self.mode == CultNetRudpSocketMode::Server { self.remote_addr = None; diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 161773f6..f60cf582 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -467,3 +467,118 @@ fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { } Ok(()) } + +/// The distinct reliable Accept sequences a peer has been sent so far. +fn accept_sequences(peer: &UdpSocket) -> Result> { + let mut buffer = vec![0_u8; 65_535]; + let mut sequences = std::collections::BTreeSet::new(); + while let Ok((received, _)) = peer.recv_from(&mut buffer) { + let packet = decode_rudp_packet(&buffer[..received])?; + if packet.packet_type == CultNetRudpPacketType::Accept && packet.reliable { + sequences.insert(packet.sequence); + } + } + Ok(sequences) +} + +/// A Connect that repeats (a moved flow, a retransmit, a storm) is answered +/// with the Accept already owed, never a fresh reliable Accept per Connect. +#[test] +fn hub_owes_one_reliable_accept_however_many_connects_repeat() -> Result<()> { + let mut hub = + CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + let connect = peer_session.create_connect(0, b"peer".to_vec())?; + for _ in 0..20 { + send_to(&peer, hub_addr, &connect)?; + while hub.receive_event_once()?.is_some() {} + } + assert_eq!(accept_sequences(&peer)?.len(), 1); + Ok(()) +} + +#[test] +fn server_mode_owes_one_reliable_accept_however_many_connects_repeat() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + let connect = peer_session.create_connect(0, b"peer".to_vec())?; + for _ in 0..20 { + send_to(&peer, server_addr, &connect)?; + let _ = server.receive_once()?; + } + assert!(server.outstanding_reliable_packet_count() <= 1); + Ok(()) +} + +/// A late duplicate Connect resets what the server learned of the peer, never +/// what it already issued: a reused sequence is dropped by the peer as a +/// duplicate. +#[test] +fn server_mode_never_reuses_a_sequence_after_a_late_duplicate_connect() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + let connect = peer_session.create_connect(0, b"peer".to_vec())?; + send_to(&peer, server_addr, &connect)?; + let _ = server.receive_once()?; + server.send_reliable("schema", b"before".to_vec())?; + send_to(&peer, server_addr, &connect)?; + let _ = server.receive_once()?; + server.send_reliable("schema", b"after".to_vec())?; + + let mut buffer = vec![0_u8; 65_535]; + let mut delivered = Vec::new(); + while let Ok((received, _)) = peer.recv_from(&mut buffer) { + let result = peer_session.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + delivered.extend(result.delivered.into_iter().map(|frame| frame.payload)); + } + assert!(delivered.contains(&b"before".to_vec())); + assert!( + delivered.contains(&b"after".to_vec()), + "the frame sent after the duplicate Connect was dropped as a reused sequence" + ); + Ok(()) +} + +/// A session that ended forgot its unacknowledged writes; a flush that then +/// reports success would claim writes the peer never acknowledged. +#[test] +fn a_write_unacknowledged_when_the_session_is_refused_fails_its_flush() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server.recv_from(&mut buffer)?; + let mut server_session = raw_session(CONNECTION_ID); + let accept = + server_session.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + let receipt = client.send_reliable("schema", b"never acknowledged".to_vec())?; + send_to(&server, client_addr, &poisoned(&mut server_session)?)?; + let _ = client.receive_once()?; + assert!(client.disconnect_reason().is_some()); + + let flushed = client.flush_reliable(Duration::from_millis(200)); + assert!(flushed.is_err(), "the write was never acknowledged"); + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated + ); + Ok(()) +} From 34054db03ac136e609025779596091967aee3c59 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:37:56 +0200 Subject: [PATCH 49/64] cultnet-py: an ended session fails its flush; a repeated Connect owes one Accept flush_reliable raised nothing for writes the ended session had forgotten; it now raises ConnectionError with the reason. Server mode answered every repeated Connect by queueing a fresh reliable Accept; a Connect from the peer already accepted is now answered with the Accept still owed and queues nothing, as the Rust hub does. Co-Authored-By: Claude Opus 5.5 --- .../cultnet-py/src/cultnet_py/transport.py | 41 ++++++++++++- packages/cultnet-py/tests/test_cultnet.py | 57 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 3462b047..0f378613 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -505,6 +505,25 @@ class CultNetRudpSession: self._track_reliable(response, now_ms) return response + def answer_repeated_connect(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpPacket: + """Answers a Connect from a peer this session already accepted. + + The packet's sequence is remembered and nothing is queued, so a Connect + storm cannot grow the reliable queue. The reply is the Accept still awaiting + acknowledgement, or an Ack once it was acknowledged. + """ + self._require_connection(packet) + if packet.packet_type != CultNetRudpPacketType.CONNECT: + raise ValueError(f"Expected RUDP connect packet, got {packet.packet_type.value}") + self._apply_acknowledgements(packet) + self._remember_received(packet.sequence) + self._last_received_at_ms = now_ms + for pending in self._pending_reliable.values(): + if pending.packet.packet_type == CultNetRudpPacketType.ACCEPT: + pending.last_sent_at_ms = now_ms + return pending.packet + return self.create_ack() + def send( self, channel_id: str, @@ -975,6 +994,7 @@ class CultNetRudpSocketTransportConnection: def connect(self, payload: bytes = b"") -> None: if self.mode != CultNetRudpSocketMode.CLIENT: raise ValueError("Only a client RUDP socket transport can initiate connect") + self.disconnect_reason = None self._send_packet(self.session.create_connect(_now_ms(), payload)) def send(self, channel_id: str, payload: bytes) -> None: @@ -1024,12 +1044,18 @@ class CultNetRudpSocketTransportConnection: return None if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: + # A Connect from the peer this transport already accepted repeats: + # answer it with the Accept already owed and queue nothing. try: - accept = self.session.accept_connect(packet, _now_ms()) + if self.session.connected: + reply = self.session.answer_repeated_connect(packet, _now_ms()) + else: + reply = self.session.accept_connect(packet, _now_ms()) + self.disconnect_reason = None except ValueError: self._packets_dropped += 1 return None - self._send_packet(accept) + self._send_packet(reply) return None try: @@ -1080,7 +1106,16 @@ class CultNetRudpSocketTransportConnection: self._delivered_frames.clear() self.socket.settimeout(poll_timeout) try: - while self.session.outstanding_reliable_packet_count > 0: + while True: + # An ended session forgot its unacknowledged writes; reporting + # them flushed would be a lie. + if self.disconnect_reason is not None: + raise ConnectionError( + "RUDP session ended before its reliable writes were acknowledged: " + + self.disconnect_reason.decode("utf-8", errors="replace") + ) + if self.session.outstanding_reliable_packet_count == 0: + return if time.monotonic() >= deadline: raise TimeoutError( "RUDP reliable flush timed out with " diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 5c2680ef..57c24421 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -1079,6 +1079,63 @@ class CultNetTests(unittest.TestCase): peer_socket.close() server.close() + def _server_mode_pair(self, connection_id: int): + server_socket = bind_udp_socket() + peer_socket = bind_udp_socket() + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + ) + ) + peer = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) + + def to_server(packet) -> None: + peer_socket.sendto(encode_rudp_packet(packet), server_socket.getsockname()) + + def drain_peer() -> list: + packets = [] + while True: + try: + wire, _ = peer_socket.recvfrom(65535) + except TimeoutError: + return packets + packets.append(decode_rudp_packet(wire)) + + return server, server_socket, peer_socket, peer, to_server, drain_peer + + def test_cultnet_rudp_flush_fails_for_a_write_the_ended_session_forgot(self) -> None: + server, _, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x10203054) + try: + to_server(peer.create_connect(0, b"join")) + server.receive_once() + peer.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + server.send("schema", b"never acknowledged") + (frame,) = peer.send_many("schema", b"poison", CultNetRudpSendOptions(reliable=True, ordered=True)) + to_server(replace(frame, fragment_count=2, fragment_id=0)) + self.assertIsNone(server.receive_once()) + with self.assertRaises(ConnectionError): + server.flush_reliable(0.2) + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_owes_one_reliable_accept_however_many_connects_repeat(self) -> None: + server, _, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x10203055) + try: + connect = peer.create_connect(0, b"join") + for _ in range(20): + to_server(connect) + server.receive_once() + accepts = {p.sequence for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT} + self.assertEqual(len(accepts), 1) + self.assertLessEqual(server.outstanding_reliable_packet_count, 1) + finally: + peer_socket.close() + server.close() + def test_cultnet_rudp_session_rejects_limits_that_cannot_admit_a_connect(self) -> None: with self.assertRaises(ValueError): CultNetRudpSession(CultNetRudpSessionOptions(connection_id=1, initial_sequence=0xFFFFFFFF)) From 4814aea8b07a1e8f38c5891df8ae6333c4221e87 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:41:22 +0200 Subject: [PATCH 50/64] cultnet-ts: an ended session fails its flush; abandoned operation sessions idle out; a repeated Connect owes one Accept flush now rejects for writes an ended session forgot. acceptConnect on a connected session delegates to the new answerRepeatedConnect, so a repeated Connect re-sends the Accept still owed, or an Ack once it was acknowledged, and queues nothing; the operation service no longer builds a session per Connect. The operation service dropped sessions never, so 64 abandoned clients locked every new one out: a session idle for sessionIdleTimeoutMs (default 30 s, the document server's session_idle_timeout) is now dropped. A request whose handler rejects ends its session like a transport refusal, so the invoker learns promptly and a retransmit is not acknowledged. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/operation-service.ts | 71 +++++++--- packages/cultnet-ts/src/rudp.ts | 63 +++++++-- packages/cultnet-ts/test/cultnet.test.ts | 134 +++++++++++++++++++ 3 files changed, 237 insertions(+), 31 deletions(-) diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 3ba3634e..433c762d 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -19,6 +19,8 @@ import { CultNetPeer } from "./peer"; const DEFAULT_CONNECTION_ID = 0x43554c54; /** Matches the document server's default session cap. A Connect past it is dropped. */ const MAX_OPERATION_SESSIONS = 64; +/** Matches the document server's default `session_idle_timeout` (30 s). */ +const DEFAULT_SESSION_IDLE_TIMEOUT_MS = 30_000; export interface CultNetOperationServerOptions { runtimeId: string; @@ -26,6 +28,8 @@ export interface CultNetOperationServerOptions { port?: number; connectionId?: number; maxFragmentBytes?: number; + /** A session that has sent nothing for this long is dropped, freeing its slot. Default 30 000. */ + sessionIdleTimeoutMs?: number; handler: (request: CultNetOperationRequestMessage) => CultNetOperationResponseMessage | Promise; } @@ -72,7 +76,14 @@ export async function startCultNetOperationServer( ); }); await bindSocket(socket, options.port ?? 0, options.host ?? "127.0.0.1"); + const idleTimeoutMs = options.sessionIdleTimeoutMs ?? DEFAULT_SESSION_IDLE_TIMEOUT_MS; const resendTimer = setInterval(() => { + const now = Date.now(); + for (const [key, peer] of sessions) { + // An abandoned client must not hold a slot forever, or the cap locks + // every new client out. + if (peer.session.checkTimeout(now, idleTimeoutMs)) sessions.delete(key); + } for (const peer of sessions.values()) { for (const packet of peer.session.dueResends(Date.now())) sendPacket(peer.remote, packet); } @@ -149,9 +160,13 @@ async function handleServerDatagram( const key = `${remote.address}:${remote.port}`; let peer = sessions.get(key); if (packet.packetType === "connect") { - if (!peer && sessions.size >= MAX_OPERATION_SESSIONS) return false; - peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote }; - sessions.set(key, peer); + if (!peer) { + if (sessions.size >= MAX_OPERATION_SESSIONS) return false; + peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote }; + sessions.set(key, peer); + } + // A Connect from an admitted peer repeats: the session answers with the + // Accept already owed and queues nothing. sendPacket(remote, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); return true; } @@ -160,12 +175,7 @@ async function handleServerDatagram( try { result = peer.session.receive(packet, Date.now()); } catch { - // receive() has already recorded the packet's reliable sequence, so the - // session cannot be kept: end it and tell the peer. The reset comes first, - // or the goodbye's ack field would acknowledge the refused frame. - sessions.delete(key); - peer.session.resetPeerState(); - sendPacket(remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + endSession(sessions, key, peer, sendPacket); return false; } if (result.reply) sendPacket(remote, result.reply); @@ -176,16 +186,23 @@ async function handleServerDatagram( } for (const frame of result.delivered) { if (frame.channelId !== "schema") continue; - const message = parseCultNetMessage(decode(frame.payload)); - if (message.schemaVersion !== "cultnet.operation_request.v0") continue; - const response = await options.handler(message); - const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); - for (const responsePacket of peer.session.sendMany("schema", payload, { - reliable: true, - ordered: true, - nowMs: Date.now(), - maxFragmentBytes: options.maxFragmentBytes ?? 2048, - })) sendPacket(remote, responsePacket); + try { + const message = parseCultNetMessage(decode(frame.payload)); + if (message.schemaVersion !== "cultnet.operation_request.v0") continue; + const response = await options.handler(message); + const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); + for (const responsePacket of peer.session.sendMany("schema", payload, { + reliable: true, + ordered: true, + nowMs: Date.now(), + maxFragmentBytes: options.maxFragmentBytes ?? 2048, + })) sendPacket(remote, responsePacket); + } catch { + // The session recorded this request's sequence, so keeping it would + // acknowledge the retransmit of a request that was never handled. + endSession(sessions, key, peer, sendPacket); + return false; + } } if (packet.packetType === "data" || result.delivered.length > 0) { sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); @@ -193,6 +210,22 @@ async function handleServerDatagram( return true; } +/** + * Ends a session that took a packet it could not serve, and tells the peer. The + * reset comes first, or the goodbye's ack field would acknowledge the very + * packet that was refused. A session already replaced under the same key is left. + */ +function endSession( + sessions: Map, + key: string, + peer: RemoteSession, + sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, +): void { + if (sessions.get(key) === peer) sessions.delete(key); + peer.session.resetPeerState(); + sendPacket(peer.remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); +} + function parseRudpEndpoint(endpoint: string): { host: string; port: number } { const url = new URL(endpoint); if (url.protocol !== "rudp:") throw new Error(`CultNet operation endpoint must use rudp://: ${endpoint}`); diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 0f1cbc33..1dc4f032 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -245,16 +245,9 @@ export class CultNetRudpSession { throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); } - const pendingAccept = [...this.#pendingReliable.values()] - .find(pending => pending.packet.packetType === "accept"); - if (pendingAccept) { - this.#rememberReceived(packet.sequence); - this.#lastReceivedAtMs = nowMs; - pendingAccept.lastSentAtMs = nowMs; - return { - ...pendingAccept.packet, - payload: new Uint8Array(pendingAccept.packet.payload ?? new Uint8Array()), - }; + // A Connect from a peer this session already accepted repeats. + if (this.#connected) { + return this.answerRepeatedConnect(packet, nowMs); } this.#ensureReliableCapacity(1); this.#rememberReceived(packet.sequence); @@ -271,6 +264,32 @@ export class CultNetRudpSession { return response; } + /** + * Answers a Connect from a peer this session has already accepted. The packet's + * sequence is remembered and nothing is queued, so a Connect storm cannot grow + * the reliable queue. The reply is the Accept still awaiting acknowledgement, + * or an Ack once it was acknowledged. + */ + answerRepeatedConnect(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpPacket { + this.#requireConnection(packet); + if (packet.packetType !== "connect") { + throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); + } + this.#applyAcknowledgements(packet); + this.#rememberReceived(packet.sequence); + this.#lastReceivedAtMs = nowMs; + const pendingAccept = [...this.#pendingReliable.values()] + .find(pending => pending.packet.packetType === "accept"); + if (!pendingAccept) { + return this.createAck(); + } + pendingAccept.lastSentAtMs = nowMs; + return { + ...pendingAccept.packet, + payload: new Uint8Array(pendingAccept.packet.payload ?? new Uint8Array()), + }; + } + send( channelId: string, payload: Uint8Array, @@ -805,6 +824,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; + /** Why the session ended, while it is over; cleared by the next Connect. */ + #endedReason: Uint8Array | undefined; readonly #stats: CultNetTransportStats = { bytesReceived: 0, bytesSent: 0, @@ -866,6 +887,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement if (this.#mode !== "client") { throw new Error("Only a client RUDP socket transport can initiate connect."); } + this.#endedReason = undefined; this.#sendPacket(this.#session.createConnect(Date.now(), payload)); } @@ -883,7 +905,17 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement async flush(timeoutMs = 30_000): Promise { const deadline = Date.now() + Math.max(0, timeoutMs); - while (this.#session.outstandingReliablePacketCount > 0) { + for (;;) { + // An ended session forgot its unacknowledged writes; reporting them + // flushed would be a lie. + if (this.#endedReason !== undefined) { + throw new Error( + `RUDP session ended before its reliable writes were acknowledged: ${Buffer.from(this.#endedReason).toString("utf8")}`, + ); + } + if (this.#session.outstandingReliablePacketCount === 0) { + return; + } if (this.#closed) { throw new Error("RUDP transport closed before reliable packets were acknowledged."); } @@ -929,6 +961,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); + this.#endedReason = reason; try { this.#sendPacket(this.#session.createDisconnect(reason)); } catch { @@ -981,10 +1014,15 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } if (this.#mode === "server" && packet.packetType === "connect") { - this.#session.resetPeerState(); + // A repeated Connect from the accepted peer resets nothing: the session + // answers it with the Accept already owed. + if (!this.#session.connected) { + this.#session.resetPeerState(); + } let accept: CultNetRudpPacket; try { accept = this.#session.acceptConnect(packet, Date.now()); + this.#endedReason = undefined; } catch { this.#stats.packetsDropped += 1; return; @@ -1022,6 +1060,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } satisfies CultNetTransportFrame); } if (result.disconnected) { + this.#endedReason = result.disconnectReason ?? new Uint8Array(); this.emit("disconnect", { reason: result.disconnectReason ?? new Uint8Array() }); this.emit("close"); return; diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 9761b37f..0a513cbf 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2038,6 +2038,14 @@ test("operation service drops and counts a packet its session refuses and keeps const [notMessagePack] = garbage.sendMany("schema", Buffer.from([0xc1]), { reliable: true, ordered: true, nowMs: 0 }); toServer(notMessagePack!); await waitFor(() => server.packetsDropped === 2, "the unhandled frame dropped"); + // The session that could not serve it ends and the client is told, so its + // retransmit is not acknowledged for a request that was never handled. + await waitFor(() => received.some((p) => p.packetType === "disconnect"), "the goodbye"); + const beforeRetransmit = received.length; + toServer(notMessagePack!); + await waitFor(() => server.packetsDropped === 3, "the retransmit dropped"); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(received.length, beforeRetransmit, "the retransmit of an unhandled request was answered"); const response = await invokeCultNetOperation(server.endpoint, { schemaVersion: "cultnet.operation_request.v0", @@ -2076,3 +2084,129 @@ test("operation service drops a Connect once its session table is full", async ( await server.close(); } }); + +test("rudp flush fails for a write the ended session forgot", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x10203056; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const peer = new CultNetRudpSession({ connectionId }); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + try { + toServer(peer.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the Accept"); + peer.receive(received.find((p) => p.packetType === "accept")!, 0); + server.send("schema", Buffer.from("never acknowledged")); + toServer(poisonedFrame(peer)); + await waitFor(() => server.stats.packetsDropped === 1, "the refusal"); + await assert.rejects(server.flush(200), /ended before its reliable writes were acknowledged/); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("a Connect that repeats after its Accept was acknowledged is answered with an Ack and queues nothing", () => { + const connectionId = 0x10203057; + const client = new CultNetRudpSession({ connectionId }); + const server = new CultNetRudpSession({ connectionId }); + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 10); + client.receive(accept, 11); + server.receive(client.createAckForReceived(accept.sequence), 12); + assert.equal(server.outstandingReliablePacketCount, 0); + for (let attempt = 0; attempt < 20; attempt += 1) { + assert.equal(server.acceptConnect(connect, 20 + attempt).packetType, "ack"); + } + assert.equal(server.outstandingReliablePacketCount, 0); +}); + +test("server-mode transport and operation service owe one reliable Accept however many Connects repeat", async () => { + const connectionId = 0x43554c54; + const serverSocket = await bindUdpSocket(); + const transport = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + const service = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: () => { throw new Error("no requests expected"); }, + }); + const raw = await bindUdpSocket(); + const accepts = new Set(); + raw.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "accept" && packet.reliable) accepts.add(packet.sequence); + }); + try { + const connect = new CultNetRudpSession({ connectionId }).createConnect(0); + for (const port of [udpPort(serverSocket), Number(new URL(service.endpoint).port)]) { + accepts.clear(); + for (let attempt = 0; attempt < 20; attempt += 1) raw.send(encodeRudpPacket(connect), port, "127.0.0.1"); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(accepts.size, 1, `reliable Accepts sent by port ${port}`); + } + assert.ok(transport.outstandingReliablePacketCount <= 1); + } finally { + raw.close(); + transport.close(); + await service.close(); + } +}); + +test("operation service frees the slots of abandoned sessions once they idle out", async () => { + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + sessionIdleTimeoutMs: 300, + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }), + }); + const port = Number(new URL(server.endpoint).port); + const sockets: Socket[] = []; + const request: CultNetOperationRequestMessage = { + schemaVersion: "cultnet.operation_request.v0", + messageId: "after-abandonment", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }; + try { + for (let index = 0; index < 64; index += 1) { + const socket = await bindUdpSocket(); + sockets.push(socket); + socket.send(encodeRudpPacket(new CultNetRudpSession({ connectionId: 0x43554c54 }).createConnect(0)), port, "127.0.0.1"); + } + await new Promise((resolve) => setTimeout(resolve, 60)); + await assert.rejects(invokeCultNetOperation(server.endpoint, request, { runtimeId: "eve-test", timeoutMs: 100 })); + await new Promise((resolve) => setTimeout(resolve, 400)); + const response = await invokeCultNetOperation(server.endpoint, request, { runtimeId: "eve-test" }); + assert.equal(response.messageId, "after-abandonment"); + } finally { + for (const socket of sockets) socket.close(); + await server.close(); + } +}); From ca18dfac552cee629347d4436034f16e3e194e08 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:41:23 +0200 Subject: [PATCH 51/64] GameCult.Networking: an ended session fails its flush; a repeated Connect owes one Accept FlushReliable throws for writes an ended session forgot. A server-mode transport and the listener answer a Connect from the peer they already hold with AnswerRepeatedConnect: the Accept still owed, or an Ack, queueing nothing. The listener replaced the peer session on every Connect, reusing sequences and never reporting PeerDisconnected; it now replaces only on a different connect payload and reports the old peer, as the Rust hub does. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 83 +++++++++++++++++- .../NetworkingTests.cs | 85 +++++++++++++++++++ 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 03543ffe..307f7aa8 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1025,6 +1025,36 @@ namespace GameCult.Networking return response; } + /// + /// Answers a connect from a peer this session has already accepted. The packet's sequence is + /// remembered and nothing is queued, so a connect storm cannot grow the reliable queue. The reply + /// is the accept still awaiting acknowledgement, or an ack once it was acknowledged. + /// + public CultNetRudpPacket AnswerRepeatedConnect(CultNetRudpPacket packet, long nowMs = 0) + { + RequireConnection(packet); + if (packet.PacketType != CultNetRudpPacketType.Connect) + { + throw new InvalidOperationException($"Expected RUDP connect packet, got {packet.PacketType}."); + } + + ApplyAcknowledgements(packet); + RememberReceived(packet.Sequence); + _lastReceivedAtMs = nowMs; + lock (_pendingReliableGate) + { + foreach (var pending in _pendingReliable.Values) + { + if (pending.Packet.PacketType != CultNetRudpPacketType.Accept) + continue; + pending.LastSentAtMs = nowMs; + return ClonePacket(pending.Packet); + } + } + + return CreateAck(); + } + /// /// Creates a data packet. /// @@ -1715,6 +1745,7 @@ namespace GameCult.Networking public sealed class CultNetRudpSocketTransportConnection : IDisposable { private const int MinimumReceiveBufferBytes = 4 * 1024 * 1024; + private static readonly byte[] ReplacedPeerReason = Encoding.UTF8.GetBytes("replaced by a new Connect generation"); // One acknowledgement carries the preceding 32-packet receive mask. // Pace at that transport window instead of sleeping after every fragment; // a one-millisecond sleep can consume a full scheduler quantum on Windows. @@ -1810,6 +1841,7 @@ namespace GameCult.Networking throw new InvalidOperationException("Only a client RUDP socket transport can initiate connect."); } + DisconnectReason = null; lock (_sessionGate) SendPacket(_session.CreateConnect(NowMs(), payload ?? Array.Empty())); } @@ -1890,6 +1922,12 @@ namespace GameCult.Networking { while (true) { + // An ended session forgot its unacknowledged writes; reporting them + // flushed would be a lie. + var ended = DisconnectReason; + if (ended != null) + throw new InvalidOperationException( + "RUDP session ended before its reliable writes were acknowledged: " + Encoding.UTF8.GetString(ended)); lock (_sessionGate) { if (_session.OutstandingReliablePacketCount == 0) @@ -2186,7 +2224,17 @@ namespace GameCult.Networking CultNetRudpPacket accept; try { - accept = _session.AcceptConnect(packet, NowMs()); + // A connect from the peer this transport already accepted repeats: answer it + // with the accept already owed and queue nothing. + if (_session.Connected) + { + accept = _session.AnswerRepeatedConnect(packet, NowMs()); + } + else + { + accept = _session.AcceptConnect(packet, NowMs()); + DisconnectReason = null; + } } catch (InvalidOperationException) { @@ -2390,12 +2438,14 @@ namespace GameCult.Networking /// public sealed class CultNetRudpSocketServerPeer { - internal CultNetRudpSocketServerPeer(EndPoint remoteEndPoint, CultNetRudpSession session) + internal CultNetRudpSocketServerPeer(EndPoint remoteEndPoint, byte[] connectPayload, CultNetRudpSession session) { RemoteEndPoint = remoteEndPoint; + ConnectPayload = connectPayload; Session = session; } + internal byte[] ConnectPayload { get; } internal CultNetRudpSession Session { get; } internal object SessionGate { get; } = new object(); @@ -2616,8 +2666,32 @@ namespace GameCult.Networking var peerKey = RemoteKey(remote); if (packet.PacketType == CultNetRudpPacketType.Connect) { + var connectPayload = packet.Payload ?? Array.Empty(); + _peers.TryGetValue(peerKey, out var admitted); + // The same payload from an admitted peer is a repeated Connect: answer it with the + // Accept already owed and queue nothing. A different payload is a new generation. + if (admitted != null && admitted.ConnectPayload.AsSpan().SequenceEqual(connectPayload)) + { + lock (admitted.SessionGate) + { + CultNetRudpPacket reply; + try + { + reply = admitted.Session.AnswerRepeatedConnect(packet, NowMs()); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } + SendPacket(admitted.RemoteEndPoint, reply); + } + return true; + } + var peer = new CultNetRudpSocketServerPeer( CloneEndPoint(remote), + connectPayload, new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = _connectionId, @@ -2628,6 +2702,11 @@ namespace GameCult.Networking _peers[peerKey] = peer; lock (peer.SessionGate) SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + if (admitted != null) + { + admitted.DisconnectReason = ReplacedPeerReason; + PeerDisconnected?.Invoke(admitted); + } return true; } diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index ef4d5a3d..ce39ea30 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1438,6 +1438,91 @@ namespace GameCult.Networking.Tests Assert.That(DrainPackets(peerSocket), Is.Empty); } + [Test] + public void RudpFlushReliable_FailsForAWriteTheEndedSessionForgot() + { + using var serverSocket = BindUdpSocket(); + using var peerSocket = BindUdpSocket(); + var serverEndPoint = serverSocket.LocalEndPoint!; + const uint connectionId = 0x10203058; + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = connectionId + }); + var peer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }); + + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(peer.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Receive(DrainPackets(peerSocket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + server.Send("schema", Encoding.UTF8.GetBytes("never acknowledged")); + + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(PoisonedFrame(peer)), serverEndPoint); + Assert.That(server.ReceiveOnce(), Is.Null); + Assert.That(server.DisconnectReason, Is.Not.Null); + Assert.Throws(() => server.FlushReliable(TimeSpan.FromMilliseconds(200))); + } + + [Test] + public void RudpServerMode_OwesOneReliableAcceptHoweverManyConnectsRepeat() + { + using var serverSocket = BindUdpSocket(); + using var peerSocket = BindUdpSocket(); + const uint connectionId = 0x10203059; + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = connectionId + }); + var connect = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }).CreateConnect(0); + for (var attempt = 0; attempt < 20; attempt++) + { + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(connect), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + } + + var accepts = DrainPackets(peerSocket).Where(p => p.PacketType == CultNetRudpPacketType.Accept && p.Reliable); + Assert.That(accepts.Select(p => p.Sequence).Distinct().Count(), Is.EqualTo(1)); + } + + [Test] + public void RudpListener_OwesOneReliableAcceptPerGenerationAndReportsAReplacedPeer() + { + using var serverSocket = BindUdpSocket(); + using var peerSocket = BindUdpSocket(); + const uint connectionId = 0x1020305a; + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = connectionId + }); + var ended = new List(); + server.PeerDisconnected += ended.Add; + var connect = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }).CreateConnect(0, Encoding.UTF8.GetBytes("join")); + for (var attempt = 0; attempt < 20; attempt++) + { + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(connect), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + } + + var accepts = DrainPackets(peerSocket).Where(p => p.PacketType == CultNetRudpPacketType.Accept && p.Reliable); + Assert.That(accepts.Select(p => p.Sequence).Distinct().Count(), Is.EqualTo(1)); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(ended, Is.Empty); + + // A different payload from the same endpoint is a new generation, and the old one is reported. + var other = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = connectionId }).CreateConnect(0, Encoding.UTF8.GetBytes("rejoin")); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(other), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(ended, Has.Count.EqualTo(1)); + } + [Test] public void RudpConfiguration_RejectsLimitsThatCannotAdmitAConnect() { From afd52e97fa6d168acc31b42fb98313a951b77f5a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:44:32 +0200 Subject: [PATCH 52/64] GameCult.Networking: the replaced-peer reason belongs to the listener that reports it Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 307f7aa8..e98fdc03 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1745,7 +1745,6 @@ namespace GameCult.Networking public sealed class CultNetRudpSocketTransportConnection : IDisposable { private const int MinimumReceiveBufferBytes = 4 * 1024 * 1024; - private static readonly byte[] ReplacedPeerReason = Encoding.UTF8.GetBytes("replaced by a new Connect generation"); // One acknowledgement carries the preceding 32-packet receive mask. // Pace at that transport window instead of sleeping after every fragment; // a one-millisecond sleep can consume a full scheduler quantum on Windows. @@ -2488,6 +2487,7 @@ namespace GameCult.Networking public sealed class CultNetRudpSocketTransportServer : IDisposable { private const int MinimumReceiveBufferBytes = 4 * 1024 * 1024; + private static readonly byte[] ReplacedPeerReason = Encoding.UTF8.GetBytes("replaced by a new Connect generation"); // Keep server response pacing aligned with the 32-packet ACK mask. Large // content responses otherwise pay one Windows scheduler quantum per 1 KiB. private const int WireBurstPackets = 32; From 9475e3653d3e870382cef098262a915c8b968da2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:47:53 +0200 Subject: [PATCH 53/64] cultnet-rs, cultnet-ts: tests that fail when the fix is removed The hub test counts replaced sessions, since a fresh session repeats the same Accept sequence and the distinct-sequence count could not see it. The TS test requires an Ack, not a new Accept, once the Accept was acknowledged. A peer Disconnect is pinned as an ended session for flush and receipts. Co-Authored-By: Claude Opus 5.5 --- .../cultnet-rs/tests/rudp_stray_packets.rs | 40 ++++++++++++++++++- packages/cultnet-ts/test/cultnet.test.ts | 30 +++++++++----- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index f60cf582..bbb96395 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -491,10 +491,18 @@ fn hub_owes_one_reliable_accept_however_many_connects_repeat() -> Result<()> { let peer = socket()?; let mut peer_session = raw_session(CONNECTION_ID); let connect = peer_session.create_connect(0, b"peer".to_vec())?; + let mut connected = 0; for _ in 0..20 { send_to(&peer, hub_addr, &connect)?; - while hub.receive_event_once()?.is_some() {} + while let Some(event) = hub.receive_event_once()? { + assert!( + matches!(event, CultNetRudpServerEvent::Connected { .. }), + "a repeated Connect must not replace the admitted session" + ); + connected += 1; + } } + assert_eq!(connected, 1); assert_eq!(accept_sequences(&peer)?.len(), 1); Ok(()) } @@ -582,3 +590,33 @@ fn a_write_unacknowledged_when_the_session_is_refused_fails_its_flush() -> Resul ); Ok(()) } + +/// A peer's Disconnect ends the session as surely as a refusal: writes it never +/// acknowledged are lost, and the transport says so. +#[test] +fn a_write_unacknowledged_when_the_peer_disconnects_fails_its_flush() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server.recv_from(&mut buffer)?; + let mut server_session = raw_session(CONNECTION_ID); + let accept = + server_session.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + let receipt = client.send_reliable("schema", b"never acknowledged".to_vec())?; + send_to(&server, client_addr, &server_session.create_disconnect(b"bye".to_vec()))?; + let _ = client.receive_once()?; + + assert!(client.flush_reliable(Duration::from_millis(200)).is_err()); + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated + ); + Ok(()) +} diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 0a513cbf..cf4af8c0 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2145,20 +2145,30 @@ test("server-mode transport and operation service owe one reliable Accept howeve handler: () => { throw new Error("no requests expected"); }, }); const raw = await bindUdpSocket(); - const accepts = new Set(); - raw.on("message", (wire) => { - const packet = decodeRudpPacket(wire); - if (packet.packetType === "accept" && packet.reliable) accepts.add(packet.sequence); - }); + let received: CultNetRudpPacket[] = []; + raw.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const settle = () => new Promise((resolve) => setTimeout(resolve, 100)); try { - const connect = new CultNetRudpSession({ connectionId }).createConnect(0); for (const port of [udpPort(serverSocket), Number(new URL(service.endpoint).port)]) { - accepts.clear(); + const peer = new CultNetRudpSession({ connectionId }); + const connect = peer.createConnect(0); + received = []; for (let attempt = 0; attempt < 20; attempt += 1) raw.send(encodeRudpPacket(connect), port, "127.0.0.1"); - await new Promise((resolve) => setTimeout(resolve, 100)); - assert.equal(accepts.size, 1, `reliable Accepts sent by port ${port}`); + await settle(); + const accepts = received.filter((p) => p.packetType === "accept"); + assert.equal(new Set(accepts.map((p) => p.sequence)).size, 1, `reliable Accepts sent by port ${port}`); + + // Once the Accept is acknowledged, a repeated Connect is answered with an Ack, never a new Accept. + peer.receive(accepts[0]!, 0); + raw.send(encodeRudpPacket(peer.createAckForReceived(accepts[0]!.sequence)), port, "127.0.0.1"); + await settle(); + received = []; + for (let attempt = 0; attempt < 20; attempt += 1) raw.send(encodeRudpPacket(connect), port, "127.0.0.1"); + await settle(); + assert.equal(received.filter((p) => p.packetType === "accept").length, 0, `Accepts sent by port ${port} after the ack`); + assert.ok(received.some((p) => p.packetType === "ack"), `an Ack from port ${port}`); } - assert.ok(transport.outstandingReliablePacketCount <= 1); + assert.equal(transport.outstandingReliablePacketCount, 0); } finally { raw.close(); transport.close(); From d85877a7be324c2d0e5d577d6b666ca78d51e8bb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:54:03 +0200 Subject: [PATCH 54/64] cultnet-rs: the session no longer keeps an initial sequence nothing reads Reset used to reissue from it; issued sequences now survive a reset, so the field was dead. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-rs/src/rudp.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 55a8d8cf..29467890 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -180,7 +180,6 @@ pub struct CultNetRudpSession { max_pending_reliable_packets: Option, max_payload_bytes: Option, max_pending_fragment_sets: usize, - initial_sequence: u32, next_sequence: u32, next_sequenced_by_channel: BTreeMap, next_fragment_id: u16, @@ -208,7 +207,6 @@ impl CultNetRudpSession { max_pending_reliable_packets: options.max_pending_reliable_packets, max_payload_bytes: None, max_pending_fragment_sets: 64, - initial_sequence: options.initial_sequence, next_sequence: options.initial_sequence, next_sequenced_by_channel: BTreeMap::new(), next_fragment_id: 1, From 4252ce65aa9535adf72ae3b870301e7ee9398c8d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:16:34 +0200 Subject: [PATCH 55/64] Map: C2a Soul pass 3 and the C0-with-C2a release ruling Co-Authored-By: Claude Opus 5.5 --- docs/document-variants-cut.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index a23fe4ea..5e191df8 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -356,6 +356,19 @@ stripped (probe). Other findings: - Fix batch 3 in Hands: F2, marking by content in every runtime, F3 routed through the cache's header decision and load checks, F4. +**C2a Soul pass 3 (`9ed61dd2`), 2026-09-30: fix first.** +- A flattened record's `HoldsIds` goes stale when its base changes in the same batch. +- A conditional commit onto disk ignores content. +- Rust `push_all` clobbers headers it cannot read. +- TS keeps a stale header after the file goes away. +- Self's claim that "rollout order makes the raw-writer hole benign" was wrong. The reader that strips ids + from a v1 store is a post-C0, pre-C2a build. That is exactly the reader the v3 marker targets, and rollout + order does not keep it away. + +**Ruling, 2026-09-30 (operator): C0 is released only together with C2a.** The first CultLib release that +refuses v3 also contains C2a, so no released reader strips ids. C0 and C1 are never released on their own. +Raw and cross-runtime writers that cannot see ids stay sticky-only. Fix batch 4 is in Hands. + **Operator rulings, 2026-09-30:** - **Duplicate unique-index values do not load** ("Duplicate indices should not load"). This covers plain records too, so R6-for-indexes becomes one general rule: From 9321c98a38b0b6fc808f7e525756d62cb24164f5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:24:52 +0200 Subject: [PATCH 56/64] Map the RUDP ack cut: Received and Delivered receipts Imagination pass 1. Probes found acknowledged ordered frames lost in live sessions in every runtime; the map derives Delivered at the sender from a correct receiver watermark. Operator questions Q-A1..Q-A5 open. Co-Authored-By: Claude Opus 5.5 --- docs/cultnet-rudp-ack-cut.md | 756 +++++++++++++++++++++++++++++++++++ 1 file changed, 756 insertions(+) create mode 100644 docs/cultnet-rudp-ack-cut.md diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md new file mode 100644 index 00000000..f655e90a --- /dev/null +++ b/docs/cultnet-rudp-ack-cut.md @@ -0,0 +1,756 @@ +# CultNet RUDP: a frame is Received, then Delivered, and the receiver owns ordered delivery with one watermark + +Status: cut map, Imagination pass 1, 2026-09-30. It is not committed yet. Self +commits it to `main` as `docs/cultnet-rudp-ack-cut.md`. The ends are the operator +ruling of 2026-09-30, summarised below. This document owns the means. + +- **Body.** `origin/hands/rudp-stray-packets` at `d85877a7` ("the session no + longer keeps an initial sequence nothing reads"). Every `file:line` is against + it unless it names another revision. + - `main` (`ae8576d8`) has not touched any RUDP file since the branch forked at + `60663f95`. `git diff --stat` over the four transport files and + `packages/cultmesh-kotlin` is empty. + - Hands is still working on that branch. The line numbers are good until it + moves again; the anchors (function names) are the durable reference. +- **Base of every cut.** `main` after `hands/rudp-stray-packets` merges. This + campaign does not start before that merge, and it does not rebase that branch. +- **Scope.** The RUDP session and socket transports in all five runtimes: C# + `src/GameCult.Networking/CultNetTransport.cs`, Rust + `packages/cultnet-rs/src/rudp.rs` (plus `packages/cultmesh-rs`), TypeScript + `packages/cultnet-ts/src/rudp.ts`, Python + `packages/cultnet-py/src/cultnet_py/transport.py`, and Kotlin + `packages/cultmesh-kotlin/.../CultMesh.kt`. Kotlin was not in the brief. It is + a fifth RUDP implementation with the same ordered-delivery code + (`CultMesh.kt:2939-2978`) and the same codec. **Q-A5** decides whether it is in. +- **Wire.** This map changes no byte on the wire (§1). + + +## The ends (operator, 2026-09-30) + +- A frame the peer has buffered but not delivered must not look delivered. + Today an ordered frame that arrives behind a gap is held and acknowledged; if + the session ends it is lost, and the sender saw "Acknowledged". +- Resending stays bounded. The sender stops retransmitting when the peer has + **received** the frame, exactly as today. Delivery-only acknowledgement was + rejected: "I don't like the possibility of unbounded resending on an + unresponsive peer application. TCP sounds like it has the right idea." +- Two meanings: **Received** (the frame is in the peer's session) and + **Delivered** (handed to the peer's application). Unordered and unreliable + channels deliver on receipt. Receipts go Pending, then Received, then + Delivered, plus the existing Invalidated. +- A session that ends with frames held leaves their receipts at Received, + visibly not Delivered. + + +## The finding that shapes this map + +Self's proposal puts a second acknowledgement on the wire: a per-channel +"delivered through N" watermark beside the received ack. Probing the receivers +changed the picture in two ways. + +**1. The ordered receivers are broken in every runtime, and the break is +bigger than the session-end case.** A held frame is not only lost at session +end. In a live session it can be stranded forever, or dropped, while the sender +sees it acknowledged. The per-channel `next` pointer +(`CultNetTransport.cs:918`, `rudp.rs:194`, `rudp.ts:160`, `transport.py:424`, +`CultMesh.kt:2643`) has been patched twice (`436654de` added the +first-use initialisation, `4335ab10` added the skip over other channels' +sequences) and is still wrong: + +- **P1: a gap filled by another channel strands the ordered channel.** Ordered + `o1`, reliable `u` on another channel (lost), `o2`, `o3`. When `u` arrives, + only `u`'s channel is drained. `o2` and `o3` are never delivered, and the + sender's pending set is empty. The same holds with two ordered channels + (P1b). +- **P6: a channel first used after other traffic loses frames.** `C1`, `C2` on + a new channel, then `X` on another. `X` arrives, then `C2`, then `C1`. The + channel initialises its `next` from `min(highest+1, frame)`, so `C2` is + delivered first and `C1` is **dropped as old** (`frame.sequence < next`). It + was acknowledged. +- Both reproduce at the branch tip, at Odin's pin `a8aeddac` and at Muninn's + pin `c2a9a6e5`, in every runtime probed. + +**2. Once the receiver is correct, "Delivered" is not new information.** A +conforming receiver holds an ordered frame for exactly one reason: some +reliable sequence below it has not arrived. When the last missing sequence +arrives, the same `Receive` call drains the held frames, before the +acknowledgement is built. So: + +> An ordered frame is Delivered exactly when every reliable sequence up to its +> last one has been Received. + +The sender already knows that. It is the sender's own **acknowledged-through** +watermark: one below the lowest sequence that is still pending or queued. +TCP's cumulative ACK means the same thing, "every byte below N is in the +receive buffer", and it sits beside SACK, which means Received. Here the +sender can compute the cumulative value itself, because every sequence is +acknowledged individually and removed from its pending set. + +A delivered watermark on the wire would be a second writer of a value the +first one already determines. Old receivers would never send it, so the sender +would need the derived rule as a fallback anyway. **This map derives Delivered +at the sender and changes no wire byte.** The explicit watermark was probed +and is available as an additive option, but it is not recommended. **Q-A1** +puts it to the operator, because it departs from Self's proposal. + +The derivation is sound only if nothing the receiver acknowledged is later +dropped while the session lives. The probes found four places where it is: + +| # | Where an acknowledged frame is lost while the session lives | Runtimes | Fixed by | +|---|---|---|---| +| P1, P1b | A cross-channel gap strands the ordered channel | all five | Cut 1 | +| P6 | Late channel initialisation drops the earlier frame | all five | Cut 1 | +| P5 | A reliable fragment set is evicted after its fragments were acknowledged | Rust, TS, Python (C# and Kotlin have no eviction) | Cut 2 | +| P8 | A retransmit more than 4,096 sequences behind the receiver's highest counts as a duplicate: acknowledged and dropped | all five | Cut 3 (the sender never lets its span reach it) | + +And one place where the sender loses track without the receiver's help: + +| # | Sender-side lie | Runtime | Fixed by | +|---|---|---|---| +| P2 | An expired reliable packet leaves a permanent hole. Ordered frames after it are held forever, and `flush_reliable` succeeds (outstanding 0) | Rust (the only runtime with reliable expiry) | Cut 4 reports it honestly; **Q-A3** decides the real fix | +| P9 | A receipt for an expiring send reads `Acknowledged` once the packet expires unacknowledged | Rust | Cut 4 | +| P10 | `send_reliable` with a full window queues the frame, then returns an error ("receipts require a non-empty reliable packet set"), so a caller that retries sends it twice | Rust | Cut 4 | + + +## Probe evidence + +Everything ran on Yggdrasil through `ygg-verify.sh`, from local-only probe +branches in the throwaway worktrees `F:\Projects\CultLib-imag-ack` and +`-imag-ack-old`. Each probe commit also survives on the Yggdrasil mirror as +`refs/verify/`. + +- `de2bdfe8`, `09318d13`, `03eac6ff`: Rust session and socket probes on the + branch at `ca18dfac` (`packages/cultnet-rs/tests/probe_ack.rs`). +- `b70a69ca`: the C#, TS and Python probes (`ProbeAckTests.cs`, + `probe-ack.test.ts`, `probe_ack.py`) on `ca18dfac`. +- `2320a4e5`: the C# probes rebased on `d85877a7`. + - At `ca18dfac`, `GameCult.Networking` did not compile (`CultNetTransport.cs(2707)`: `ReplacedPeerReason` not in scope). `afd52e97` on the branch fixes it. +- `7fe5d726`: P5 and P8 in C#, TS and Python. +- `f0e7136c`: the Rust probes at Odin's pin `a8aeddac`. +- `bf8fc826`: all four runtimes' probes at Muninn's pin `c2a9a6e5`. +- `db7d2886`: **the prototype**, the Rust receiver rebuilt on one watermark + plus sender-derived receipt states (about +130/−125 in `rudp.rs`). + +| # | Probe | Branch tip (all runtimes probed) | Pins `a8aeddac` / `c2a9a6e5` | Prototype `db7d2886` | +|---|---|---|---|---| +| P1 | `o1`, reliable `u` lost, `o2`, `o3`; then `u`, then `o3` | Delivered `[o1, u]`. `o2`, `o3` never. Sender pending `[]`. Rust, C#, TS, Python. | Same in Rust (both pins), C#, TS, Python (`c2a9a6e5`) | `[o1, u, o2, o3]` | +| P1b | `A1`, `B1` (second ordered channel) lost, `A2`, `A3` | `[A1, B1]`, all four | Same | `[A1, B1, A2, A3]` | +| P6 | `C1`, `C2` on a new channel, then `X`; arrival `X`, `C2`, `C1` | `[C2]`. `C1` dropped. All four. | Same | `[C1, C2, X]` | +| P2 | Rust: reliable media with expiry 1 ms, lost; then `s1`, `s2` on `schema` | Nothing delivered. Sender outstanding 0, expired 1, no resends. `flush_reliable` would return Ok. | Same | Unchanged (held behind the hole). Cut 4 makes the sender say so. | +| P3 | `s1` lost, `s2` arrives and is acknowledged | `s2` not delivered, but acknowledged: sender pending `[s1]` only. A reset then loses `s2`. | Same | Receipt for `s2`: `Received`; after `s1` arrives, 2 frames delivered and `Delivered` | +| P4 | Rust: `s0` delivered, `s1` lost, then a 5 MiB ordered frame in 60 kB fragments, pumped with acks | `RUDP ordered hold buffer is full`: the session is refused. After the gap it would have delivered. With no gap, delivered. | (probe predates the fix; not evidence) | The prototype held every ordered frame, so a 5 MiB frame was refused **even with no gap**: hold only what is not yet deliverable (Cut 1) | +| P5 | 3-fragment reliable frame, 2 fragments arrive and are acked; 70 lossy unreliable fragment sets; last fragment arrives | Rust, TS, Python: reliable frame never delivered, sender pending `[]` | Same (Rust) | Unchanged (Cut 2) | +| P7 | An Ack with flag bit `0x10`, a nonzero sequence field, or a 5-byte payload | Decoded and applied (ack cleared the pending sequence) in Rust, C#, TS, Python | Same at both pins | n/a | +| P8 | Reliable `g` lost while 4,200+ later sequences are sent and acked; then `g` arrives | `g` not delivered, and acknowledged: the sender stops. Rust, C#, TS, Python. | n/a | Unchanged (Cut 3) | +| P9 | Rust socket: `send_reliable("media")` with expiry 1 ms, never read; a ping triggers the purge | `Pending`, then `Acknowledged` | n/a | `Pending` (the abandoned sequence is never acknowledged) | +| P10 | Rust socket: 40 `schema` sends, then `send_reliable` | Error "receipts require a non-empty reliable packet set", after the frame was queued | n/a | Not addressed by the prototype (Cut 4) | + +**Suites on the prototype `db7d2886`** (baseline `03eac6ff`, the probe commit +without the prototype, was all green): + +- `cultnet-rs`: 110 + 4 + 24 pass; `tests/cultnet.rs` 51 pass and 2 fail. + Both failures are test shape, not mechanism: + - `rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads_in_sequence` + (`tests/cultnet.rs:1000`) hands the receiver an Accept with sequence 91 from a + sender whose data starts at 1. That handshake is impossible. Under a + watermark seeded by the Accept, every data sequence is "already below" it. + The test must use a consistent handshake. + - `rudp_ordered_channel_tracking_is_bounded` (`:2655`) sends 256 ordered + channels with no gap. The prototype keeps state only for channels with held + frames, so nothing is refused. The bound becomes "channels with held + frames", and the test must hold frames behind a gap. +- `cultmesh-rs` did not compile: `lib.rs:892` matches the status enum + exhaustively and does not cover `Received`. Cut 4 rewrites that call site. +- The probe suite: P1, P1b, P6 and P3b are fixed as the table shows. + +**Wire compatibility (P7).** Every decoder ignores flag bits 4-7 and byte 35. +Every session ignores an Ack's sequence field and payload +(`CultNetTransport.cs:1177`, `rudp.rs:572-586`, `rudp.ts:397`, +`transport.py:612`, `CultMesh.kt:2738`). Every decoder rejects an unknown +packet type (`CultNetTransport.cs` `Enum.IsDefined`, `rudp.rs` +`packet_type_from_code`, `rudp.ts:1290`, `transport.py:1212`, +`CultMesh.kt` `fromCode`) and an unknown version byte. So: + +- An additive watermark fits in an Ack: flag bit 4 set, the watermark in the + Ack's unused sequence field, zero extra bytes. Old peers apply the ack and + ignore the rest, proven at both pins. +- A new packet type or a version bump is not additive: old peers refuse the + datagram, and since the stray-packet branch they count it as dropped. +- No capability negotiation is needed under this map, because nothing on the + wire changes. + + +## Authority map + +- **Owner of ordered delivery (receiver): the session's contiguous + received-through watermark `R`.** + - Every reliable sequence up to `R` has been received. + - A held ordered frame whose first sequence is at most `R + 1` is delivered, + in sequence order, in the same call that advanced `R`. + - `R` is seeded by the handshake. `AcceptConnect` seeds it with the Connect's + sequence, and receiving the Accept seeds it with the Accept's sequence. A + session connected without a handshake (Rust `assume_connected`, + `rudp.rs:241`, test-only today) seeds it from its first reliable packet. + - It advances in `RememberReceived` and resets in `ResetPeerState`. +- **Owner of Received (sender): the pending and queued sets**, as today. A + sequence leaves them when an ack names it. Retransmission stops there. +- **Owner of Delivered (sender): the acknowledged-through watermark `A`**, one + below the lowest sequence that is pending, queued or abandoned. An ordered + send is Delivered when `A` reaches its last sequence. A reliable unordered + send is Delivered when it is Received. +- **Inputs.** Received sequences (receiver); acks and expiry (sender). Nothing + else. +- **Outputs.** Delivered frames, in order (receiver). Receipt status and flush + completion (sender). +- **Derived, not owners.** + - The per-channel `next` map is **deleted**, not demoted: + `_orderedNextSequenceByChannel` (`CultNetTransport.cs:918`), + `ordered_next_sequence_by_channel` (`rudp.rs:194`), + `#orderedNextSequenceByChannel` (`rudp.ts:160`), + `_ordered_next_sequence_by_channel` (`transport.py:424`), + `orderedNextSequenceByChannel` (`CultMesh.kt:2643`). + - `expectedSequenceIfUninitialized` and its equivalents are deleted + (`CultNetTransport.cs:1151`, `rudp.rs:534`, `rudp.ts:375`, + `transport.py:595`, `CultMesh.kt:2728`). + - `PendingOrderedFrame.NextSequence` is deleted. + - `OutstandingReliablePacketCount` stays, and means "not yet Received". +- **Forbidden writers.** No path delivers an ordered frame except the drain + after `R` advances. No receive path drops a frame whose sequence it + acknowledged while the session lives. No sender path reports Delivered + except through `A`. +- **Shared paths.** Accept, reliable data, fragments, duplicates and + retransmits all advance `R` through `RememberReceived` and then call the + drain. Direct sends, fragmented sends, queued promotion and resends all feed + the same pending and queued sets that `A` reads. +- **Deletion line.** `DeliverOrdered`, `DrainOrdered` and + `SkipReceivedNonChannelSequences` and their equivalents go before the drain + is added: `CultNetTransport.cs:1628-1700`, `rudp.rs:1126-1234`, + `rudp.ts:737-800`, `transport.py:868-920`, `CultMesh.kt:2939-2978`. + + +## The design, question by question + +### 1. Where Delivered comes from + +- Derived at the sender from `A` (§ The finding). No wire change. +- **Why it is sound after Cuts 1-3.** A conforming receiver: + - drops nothing it acknowledged while the session lives (P1, P6: Cut 1; P5: + Cut 2; P8: made unreachable by Cut 3); + - holds an ordered frame only while a sequence below it is missing (Cut 1); + - drains held frames in the call that fills the last gap, before it + acknowledges (Cut 1). + - So every sequence up to `s` acknowledged means `s` was delivered. +- **Mixed versions.** A new sender with an old receiver can be wrong only when + that receiver strands a frame (P1, P6). That needs reliable traffic on more + than one channel of one session. The C#, TS, Python and Kotlin socket + transports send reliably only on `schema` (`CultNetTransport.cs:2372-2376`, + `rudp.ts:1233-1241`), so they cannot trigger it. Rust sends reliable + `media`/`audio` too (`rudp.rs` `channel_send_options`). The Eyes survey found + no live session that mixes those with `schema`: Muninn's media session + carries no `schema` traffic (`Muninn crates/muninn-daemon/src/main.rs:3099-3158`), + and neither does Sleipnir's HID stream (`Odin crates/sleipnir-daemon/src/main.rs:2217-2272`). + The old receiver loses such frames today in any case. Bumping the pin is the + fix. +- **Rejected: a wire watermark (Q-A1 B).** It is additive (P7), but it is a + second authority for a value the first one determines. Old receivers never + send it, so the derived rule is needed as a fallback anyway. + +### 2. What "delivered to the application" means + +- **Delivered means the frame left the session's hold and was handed to the + transport's application boundary.** That boundary is: + - C#: enqueued in the socket transport's `_deliveredFrames` (read by + `TryReceiveOnce`), or returned in `CultNetRudpReceiveResult.Delivered` from + a bare session; + - Rust: pushed to `delivered_frames` (`rudp.rs:2137`) or the hub's + `pending_events`, or handed to the document server's sink + (`rudp_document_server.rs:343-360`), which acknowledges only after the sink + accepted; + - TS: the `"frame"` event was emitted (`rudp.ts:1055-1061`); + - Python and Kotlin: the delivered queue. +- **It is not "consumed".** The operator rejected application-driven + acknowledgement, and TCP's cumulative ACK has the same boundary: in the + receive buffer, readable, not read. +- A listener or sink that throws or rejects has still been delivered to. The + handler-rejection paths from the stray-packet branch end the session, which + is the application's answer, not the transport's. + +### 3. The public API + +- **Rust receipts** (the only runtime that has them): + - `CultNetRudpReliableSendStatus` becomes `Pending`, `Received`, `Delivered`, + `Invalidated` (`rudp.rs:144-148`). `Acknowledged` is removed, not aliased. + Every caller must choose, and every caller today means Delivered. + - The receipt records whether its send was ordered (`rudp.rs:132-135`). + - After the session ends without a reset: `Pending` becomes `Invalidated` + (as today, `rudp.rs:1919-1928`). `Received` stays `Received`, which the + ruling asks for. `Delivered` stays. + - After a reset (`reset_peer_state`: a new Connect generation, or + `end_refused_session`), the session keeps one frozen snapshot of the ended + epoch: its scope, `A`, and its still-unacknowledged sequences, at most the + reliable window plus the queue. A receipt from that epoch resolves against + the snapshot, so `Received` stays `Received` and `Delivered` stays + `Delivered`. Older epochs are `Invalidated`. + - Today every receipt, delivered or not, reads `Invalidated` after a + reset, because the scope rotates (`rudp.rs:338-339`). + - `send_reliable` receipts cover queued packets (P10). The receipt lists every + sequence the send assigned, not only the ones admitted to the wire. + - `send_reliable` refuses a channel with reliable expiry (P9). The stale + comment at `rudp.rs:1903-1904` says this lineage has no expiry, which is + false since `bac5be88`. Restore the guard it describes. + - New: `CultNetRudpSocketTransportConnection::wait_delivered(&receipt, timeout) -> Result<()>`. + It pumps receive and resends like `flush_reliable`. It fails at once when + the session ended, naming the frozen status. Today four places hand-roll + that loop: `cultmesh-rs/src/lib.rs:806-838` and `:870-900`, + `CodexConnector src/idunn_health.rs:727-745` and + `Ghostlight crates/ghostlight-dungeon/src/idunn_health.rs:1126-1145`. +- **Flush: one wait, and it means Delivered** (**Q-A2**). + - When every sent sequence is Received, `A` is the highest sequence sent, so + every send is Delivered. The two waits coincide, except behind an abandoned + sequence. + - Rust `flush_reliable` (`rudp.rs:1985`) therefore fails at once when an + ordered send follows the session's first abandoned sequence. That case + never becomes Delivered, and today it reports success (P2). + - C# `FlushReliable` (`CultNetTransport.cs:1914`), TS `flush` + (`rudp.ts:906`), Python `flush_reliable` (`transport.py:1101`) and Kotlin + `flushReliable` (`CultMesh.kt:3186`) keep their code. Their contract text + changes to "every reliable frame sent has been delivered to the peer's + application". These runtimes have no expiry. + - A second "flush until received" variant would differ only in the + abandoned-sequence case, where it would report success on frames that + will never be delivered. It has no consumer. Not added. +- **Defaults.** Plain flush waits for Delivered. No parameter. +- **Compatibility.** No C#, TS, Python or Kotlin signature changes. Rust: + removing `Acknowledged` breaks the external callers at their next bump. Each + is a one-line change to `Delivered`, or a switch to `wait_delivered`. The + pinned callers are CodexConnector (`e171eca`) and Ghostlight (`85f7024`). + +### 4. Consumers, each assigned + +Surveyed across `F:\Projects` (the HEAD of each repo is recorded in the Eyes +report). Every consumer that waits wants **Delivered**. None wants Received +alone. + +| Consumer | What it waits on today | Assigned | Change | +|---|---|---|---| +| `cultmesh-rs/src/lib.rs:781-838` `request_raw_snapshot_from_rudp_catalog` (Idunn `src/drivers.rs:5527` uses it) | `Acknowledged`, then the response | Delivered (the response proves it anyway) | Cut 4: `wait_delivered` | +| `cultmesh-rs/src/lib.rs:870-900` `publish_cultnet_message_to_rudp_catalog` (Odin `odin-daemon/src/main.rs:616-658`) | `Acknowledged`, then disconnect. The doc at `:869` claims "application admission". | Delivered. That is true at the document server, which acknowledges after its sink accepted (`rudp_document_server.rs:343-372`). | Cut 4: `wait_delivered`; the doc says Delivered | +| `cultmesh-rs/src/lib.rs:918-938` `publish_cultnet_messages_to_rudp_catalog` (Ghostlight `mesh.rs:323`) | `flush_reliable`, then drop. **Several ordered frames, then close.** | Delivered | None; flush now means it | +| `cultmesh-ts/src/index.ts:6070-6098` `publishRudpDocumentOnce` (Heimdall, StreamPixels, Stonks, weksa, Bifrost, VoidBot, Hermodr) | `peer.flush` (`cultnet-ts/src/peer.ts:109-110`), then close | Delivered | None | +| `cultnet-ts/src/idunn-odin-presence-publisher.ts:100-127` | a hand-rolled session loop that waits for `outstandingReliablePacketCount === 0`, then closes. It never resends, and queued packets are never sent. | Delivered | Cut 5: use the socket transport and `flush` | +| `cultnet-ts/src/signed-daemon-health.ts:113-160` (Hermodr, Odin `idunn-rudp.cjs`) | a hand-rolled loop: sends only the packets `sendMany` admitted, once; waits up to 500 ms for **any** ack packet, swallows the timeout, closes | Delivered | Cut 5: use the socket transport and `flush`. Today a lost datagram is reported as published, and a payload over 32 packets is truncated. | +| `cultnet-ts/src/operation-service.ts` | the operation response | none (the response is the proof) | none | +| C# `FlushReliable` callers: Gjallar `Program.cs:3055-3080` (two ordered frames, flush, dispose) | flush | Delivered | None | +| Heimdall `src/idunn-rudp-health.ts:75-86` (vendored CultLib `b6b1d9c`) | first ack packet | Delivered | Heimdall's own cut at its bump (follow-up) | +| CodexConnector `idunn_health.rs:727-745`, Ghostlight `idunn_health.rs:1126-1145` | `Acknowledged` | Delivered | At their next pin bump | +| Muninn `main.rs:9446-9458`, Sleipnir `main.rs:880-904` (both copies) | nothing (send and drop) | none today | Follow-up: fire-and-forget health can be silently lost | +| Kotlin `flushReliable` | no callers | Delivered | Contract text only | + +### 5. The hold buffer and the 4 MiB cap + +- **It belongs here**, because the held frame is exactly the state this map + names. +- **Mechanism (P4).** Rust caps held ordered frames at 1,024 frames and 4 MiB + (`rudp.rs:47-48`, checked at `:1169-1175`). A refusal ends the session. The + default payload limit is 16 MiB (`rudp.rs:33`). With one lost packet under a + large ordered transfer, a legal 5 MiB frame is refused and kills the session. + - The sender's window counts pending packets, not span or bytes + (`rudp.rs:898-930`). While one packet stays lost, acknowledged packets keep + freeing slots, so the held data behind the gap grows without limit. +- **Fix: a sender-side flow window, TCP's receive window without a wire + field** (Cut 3). A reliable packet is admitted to the wire only while: + - its sequence is at most 1,023 above the lowest unacknowledged sequence; + and + - the payload bytes of sequences above the lowest unacknowledged one stay + within 4 MiB, with at least one packet always admissible. + - Otherwise it waits in the existing queue. +- **What this buys:** + - a conforming sender never makes a Rust receiver hold more than 4 MiB, so + the cap stops refusing legal traffic; + - a conforming sender never gets 1,024 sequences ahead of the receiver's + highest, so the Rust far-ahead refusal (`rudp.rs:46`) never drops its + packets; + - a conforming sender never leaves a sequence 4,096 behind the receiver's + highest, so P8 cannot happen. The receive-history window stays 4,096 in + every runtime (`CultNetTransport.cs:884`, `rudp.rs:35`, `rudp.ts:20`, + `transport.py:404`, `CultMesh.kt:2625`). +- **What is kept.** The cap and the far-ahead refusal stay. They protect + against hostile peers. +- **Rejected:** + - raising the cap to the payload limit or above: that is 16-64 MiB per + hostile session instead of 4; + - lowering the ordered payload limit to 4 MiB: that is an API regression for + large snapshots. +- **Cost.** Normal flow never has a span beyond the 32-packet window, so the + new limits bind only while a packet is lost. +- **Old senders** (the pinned and vendored ones) can still trip the Rust cap + until they bump. +- **Not in scope.** C#, TS, Python and Kotlin receivers have no hold cap at + all. They hold without limit for a hostile peer. Follow-up. + +### 6. Reliable expiry (P2) + +- Only Rust has it (`rudp.rs:39`, `:886-896`). An expired packet's sequence + never reaches the receiver. `R` stops below it, and so did the old `next` + pointer. Ordered frames after it are held for the rest of the session. +- **Today's users.** Only Muninn's reliable `audio` + (`Muninn main.rs:3230-3241`), and its session carries no ordered traffic. + `hands/media-fec-cut3` deletes the `audio` profile entry and the `"audio"` + send arm. The `media` channel keeps a caller-chosen reliable-with-expiry + mode. +- **This map makes it honest (Cut 4):** + - the sender records its first abandoned sequence; + - `A` never passes it; + - ordered receipts after it stay `Received`; + - `flush_reliable` fails at once. +- **The real fix is a fork (Q-A3).** The literature's answer is PR-SCTP's + FORWARD-TSN (RFC 3758): the sender tells the receiver to move its cumulative + point past abandoned sequences. That is a wire addition, and every runtime's + receiver would have to learn it. + +### 7. Interaction with the stray-packet branch + +The ack cuts build on these branch behaviours and do not alter them: + +- An ended session fails its flush (`CultNetTransport.cs:1924-1931`, + `rudp.rs:1987-2011`, `rudp.ts:906-916`, `transport.py:1101-1124`). +- A refusing session ends, and its goodbye is built after the reset + (`EndRefused`, `end_refused_session` `rudp.rs:358`). The refused sequence is + never acknowledged, so a refusal never becomes a false Received. +- Rust keeps issued sequences on reset. Cut 4's frozen epoch snapshot relies + on that: an old receipt's sequences can never collide with a new epoch's. +- One Accept per session in C#, TS and Python, and `AnswerRepeatedConnect`. A + repeated Connect's sequence is at most `R` and counts as a duplicate. +- The TS operation service expiry and handler rejection are above the + transport, and do not change. +- One thing the branch left: the Rust socket transport in server mode still + calls `reset_peer_state()` on every Connect (`rudp.rs:2099`), even a repeated + one. Under Cut 4 that rotates the epoch. That makes it visible, but it is the + branch's parity item, not this map's. It is recorded as a follow-up. + + +## Cuts + +Every cut is on CultLib, branch `hands/rudp-delivered` from `main` after +`hands/rudp-stray-packets` merges, in a worktree Self creates. Heavy +verification runs on Yggdrasil. Each cut starts by committing its probes as +failing tests named for the rule, in every runtime it touches. + +**Negative checks that stay green in every cut:** + +- the cross-runtime reorder, drop and fragment proofs in the TS interop harness + (`docs/cultnet-transport-parity.md:227-248`); +- the stray-packet tests (`rudp_stray_packets.rs`, and their C#, TS and Python + equivalents); +- the codec fixture vectors: no wire byte changes. + +### Cut 1. One watermark owns ordered delivery (receiver, all five runtimes) + +- **First.** Failing tests P1, P1b and P6 in each runtime. Rust's are in + `probe_ack.rs` at `db7d2886`, and the others are in the probe commits above. +- **Deletes first:** + - the per-channel `next` map and its resets: + - `CultNetTransport.cs:918`, `:1292-1310`; + - `rudp.rs:194`, `:221`, `:348`; + - `rudp.ts:160`, `:214-225`; + - `transport.py:424`, `:458-469`; + - `CultMesh.kt:2643`; + - `expectedSequenceIfUninitialized`: + `CultNetTransport.cs:1151-1153`, `rudp.rs:534-537`, `rudp.ts:375-377`, + `transport.py:595-599`, `CultMesh.kt:2728`; + - `DeliverOrdered`, `DrainOrdered` and `SkipReceivedNonChannelSequences` + (the deletion line above); + - `PendingOrderedFrame.NextSequence`. +- **Adds:** + - `R`, seeded in `AcceptConnect` (`CultNetTransport.cs:1012`, `rudp.rs:377`, + `rudp.ts:242`, `transport.py:485`, `CultMesh.kt:2667`) and on Accept + receipt (`CultNetTransport.cs:1155`, `rudp.rs:539`, `rudp.ts:379`, + `transport.py:601`, `CultMesh.kt:2730`); + - `R` advances in `RememberReceived` (`CultNetTransport.cs:1518`, + `rudp.rs:961`, `rudp.ts:619`, `transport.py:809`, `CultMesh.kt:2891`); + - a reassembled ordered frame is delivered at once when its first sequence + is at most `R + 1` and its channel holds nothing below it. Otherwise it is + held. The caps apply only to what is held: the prototype held everything + and so refused a 5 MiB frame with no gap; + - after every reliable receipt (data, fragment, duplicate, Accept), drain + every channel's held frames whose first sequence is at most `R + 1`, in + sequence order; + - Rust only: the 64-channel cap (`rudp.rs:42`) counts channels with held + frames. +- **Tests changed:** `tests/cultnet.rs:1000` (consistent handshake) and + `:2655` (hold frames behind a gap). The equivalents in the other runtimes are + whatever the change breaks. Each must be read before it is edited, because a + test that encodes `min(highest+1, frame)` encodes P6. +- **Verification:** + - P1, P1b and P6 pass in every runtime; + - the full suites: `cargo test` in `cultnet-rs` and `cultmesh-rs`, `dotnet + test tests/GameCult.Networking.Tests`, `npm test` in `cultnet-ts`, the + Python suite, and Kotlin per Q-A5; + - the TS interop harness across runtimes; + - negative grep: `rg -n "orderedNextSequence|ordered_next_sequence|expectedSequenceIfUninitialized|expected_sequence_if_uninitialized|SkipReceivedNonChannel|skip_received_non_channel" src packages` + returns nothing; + - mutation testing on the diff: cargo-mutants, Stryker.NET, StrykerJS, and + mutmut for Python. +- **Ledger.** Per runtime about −70 and +40. Across five, about −350 and +200. + +### Cut 2. A reliable fragment set is never evicted (receiver: Rust, TS, Python) + +- **First.** Failing test P5 in each of the three. +- **Change:** + - eviction picks the stalest set that holds no reliable fragment + (`rudp.rs:1015-1035`, `rudp.ts:679-692`, `transport.py:836-842`); + - if every pending set holds reliable fragments, the packet is refused and + the session ends, as every other refusal does. + - A conforming sender cannot reach that bound: its window keeps at most 32 + reliable packets in flight, and the bound is 64 sets. +- **Verification:** P5 delivers the reliable frame. `d3bc6e66`'s test (lossy + fragment sets keep being evicted) stays green. `fragment_sets_evicted` still + counts evictions. +- **Ledger.** About −3 and +12 per runtime. + +### Cut 3. The sender's flow window: 1,023 sequences and 4 MiB above the lowest unacknowledged (sender, all five runtimes) + +- **First.** Failing tests P8 (all runtimes) and P4 (Rust receiver, with a + sender from each runtime). +- **Change.** Admission (`AdmitReliablePackets` / `PromoteQueuedReliable` at + `CultNetTransport.cs:1439-1490`, `rudp.rs:898-930`, `rudp.ts:586-600`, + `transport.py:776-796`, `CultMesh.kt:2856-2872`) admits a packet only while: + - the window has a slot (as today); + - its sequence is at most the lowest unacknowledged sequence plus 1,023; + - the admitted payload bytes above the lowest unacknowledged sequence stay + within 4 MiB. + - The first packet is always admissible. Expired packets (Rust) do not count + as unacknowledged here: they are gone from the wire. +- **Contract.** `docs/cultnet-transport-parity.md` (after `:263`) states the + receive window: a receiver must hold 1,023 sequences and 4 MiB beyond its + contiguous watermark, and a conforming sender never exceeds that. +- **Verification:** + - P8: `g` is delivered; + - P4: the 5 MiB frame is delivered after the gap fills, and the session is + never refused; + - the existing large-snapshot tests (`c398bad1`) and the fragment interop + stay green; + - the Rust benchmark or media probe shows no throughput change on a lossless + link. +- **Ledger.** About +25 per runtime, and nothing removed. + +### Cut 4. Rust receipts: Received and Delivered, honest expiry, honest queueing (Rust, cultmesh-rs) + +- **First.** Failing tests: P3b (Received, then Delivered), P9, P10, P2 + (`flush_reliable` must fail), and the frozen-epoch case (a receipt at + `Received` stays `Received` across `end_refused_session`). +- **Deletes first:** + - `Acknowledged` (`rudp.rs:146`); + - the stale comment and the half guard at `rudp.rs:1903-1908`; + - the hand-rolled status loops at `cultmesh-rs/src/lib.rs:806-838` and + `:870-900`. +- **Adds:** + - `Received` and `Delivered`; + - `ordered` on the receipt; + - `acked_through` (`A`), reading pending, queued and first-abandoned; + - first-abandoned, recorded in `purge_expired_reliable` (`rudp.rs:886`); + - the receipt covers queued sequences: `send_many` (`:450`) reports every + sequence it assigned; + - `send_reliable` refuses a channel with expiry; + - the one-epoch frozen snapshot in `reset_peer_state` (`:338`); + - `wait_delivered`; + - `flush_reliable` fails fast after an abandoned sequence precedes an ordered + send. +- **Changes:** + - `cultmesh-rs/src/lib.rs:832` and `:892` use `wait_delivered`; + - the doc at `:869` says Delivered; + - `tests/rudp_stray_packets.rs:309` and `cultmesh-rs/tests/rudp_document_server.rs:408,415` change from `Acknowledged` to `Delivered`. +- **Verification:** + - the new tests pass; + - `rg -n "Acknowledged" packages/cultnet-rs packages/cultmesh-rs` has no hit; + - both suites are green; + - cargo-mutants on the diff. +- **Ledger.** About −60 and +110 (of which `wait_delivered` is about 35, and + it replaces two loops of about 30 each). + +### Cut 5. CultLib-internal consumers and the contract text (TS, docs) + +- **Changes:** + - `signed-daemon-health.ts:113-160` and + `idunn-odin-presence-publisher.ts:100-127` send through + `CultNetRudpSocketTransportConnection` and `flush`. Their hand-rolled + session loops are deleted. + - Flush contract text in all five runtimes: "every reliable frame sent has + been delivered to the peer's application". + - `docs/cultnet-transport-parity.md:236-240` (the control-packet guard + paragraph) is replaced by the watermark rule, and `:261-272` gains the + Received/Delivered meanings and the receive window. +- **Verification:** + - `runtime-presence-health.test.ts` and `signed-daemon-health.test.ts` stay + green; + - a new test drops the first datagram and still publishes; + - a new test publishes a payload over 32 packets whole. +- **Ledger.** About −100 and +40. + + +## Subtraction ledger (estimate) + +| Cut | Removed | Added | Other | +|---|---|---|---| +| 1 | ~350 | ~200 | five per-channel state machines become one watermark | +| 2 | ~10 | ~35 | none | +| 3 | 0 | ~125 | a documented receive-window contract | +| 4 | ~60 | ~110 | one public method (`wait_delivered`), two enum variants for one | +| 5 | ~100 | ~40 | two hand-rolled RUDP loops gone | + +- **Source, net:** about −10. Tests are extra: about 5 × 6 rule tests. +- **Wire:** zero bytes. +- **Downstream:** each external `Acknowledged` caller needs a one-line change + at its next bump. +- **Prototype check:** the Rust receiver and status prototype is +129/−124 in + `rudp.rs` and includes throwaway public helpers. + + +## Operator questions + +Each question stands alone. + +- **Q-A1. Where does Delivered come from?** + - **A.** Derive it at the sender. An ordered send is Delivered when every + reliable sequence up to its last one is Received. A reliable unordered send + is Delivered when Received. Nothing changes on the wire. + - **B.** Add a delivered watermark to the wire. The receiver sets Ack flag + bit `0x10` and puts "delivered through N" in the Ack's unused sequence + field. That is additive: old C#, Rust, TS and Python peers at the branch tip + and at both pins ignore it (P7). Senders use it when it is present, and + fall back to A for old peers. + - **Recommended: A.** + - Once Cuts 1-3 make the receiver correct, B's watermark always equals A's + derivation. + - B is a second authority for one value, plus a capability sniff, plus five + encoder and decoder changes. + - B would only tell the sender more than A if a receiver strands frames, + and that receiver is old, so it does not send B. + - **Depends on it.** Cut 4's shape. B adds a Cut 3b in all five runtimes (+~40 + each) and makes the parity codec fixtures change. +- **Q-A2. One flush, or a "received" flush beside the "delivered" one?** + - **A.** One flush. It means Delivered. + - **B.** Add `flushReceived` beside it. + - **Recommended: A.** After this map, all-Received means all-Delivered, except + behind an abandoned sequence. There a received-flush would report success + on frames that will never be delivered, and plain flush fails fast. No + consumer wants Received alone (§4). Receipts keep both states. + - **Depends on it.** B adds a method in five runtimes and a test each. +- **Q-A3. Reliable expiry next to ordered traffic (P2).** An expired reliable + packet leaves a permanent hole, and every ordered frame after it on the same + session is held until the session ends. + - **A.** Report it honestly (Cut 4: receipts stay Received and flush fails + fast). Document "do not send ordered traffic on a session that uses + reliable expiry". No live consumer does. + - **B.** PR-SCTP's FORWARD-TSN (RFC 3758). On expiry the sender sends an + abandonment notice, which rides an Ack-typed packet with flag `0x10` so + that old peers ignore it (P7). New receivers treat the sequence as received + with no frame, and acknowledge it. The sender resends the notice until it + is acknowledged. That is a new cut in all five receivers. Old receivers + never acknowledge it, so the resend needs its own bound. + - **C.** Refuse ordered sends on a session after its first expiry. + - **D.** Delete reliable expiry. Muninn's `audio` is its only user and Muninn + Q4 retires it. This subtracts, but CultLib would lose partial reliability, + which a reasonable consumer can expect. + - **Recommended: A now.** Record B as the complete answer for when a consumer + mixes the two. + - **Depends on it.** Whether Cut 4 carries `first_abandoned` alone (A) or a + new wire notice (B). +- **Q-A4. Receipts in C#, TypeScript, Python and Kotlin?** + - Only Rust has per-send receipts. The other four offer flush and an + outstanding count. + - **A.** Port receipts with the same four states and a `WaitDelivered`, in + this campaign. + - **B.** Record it as a parity gap. Flush covers every surveyed consumer. + - **Recommended: B.** No consumer outside Rust waits per send. A reasonable + C# consumer might want it, but it is additive later and does not change the + wire. + - **Depends on it.** A adds a cut of about +80 per runtime. +- **Q-A5. Is Kotlin in the campaign?** + - `cultmesh-kotlin` carries a fifth RUDP runtime with P1 and P6 (the same code, + `CultMesh.kt:2939-2978`). Its flush has no callers. The stray-packet branch + did not touch it. Its only build path is `build.ps1`, which needs Android + Studio's `kotlinc` and JBR on Windows. Yggdrasil has no Kotlin image in the + verify list. + - **A.** Include it in Cuts 1, 3 and 5, and verify on Starfire as a + Windows-only job, or add a Kotlin image to Yggdrasil first. + - **B.** Record it as parity debt, with the stray-packet gaps. Mark the parity + doc so that nobody reads Kotlin as conforming. + - **Recommended: A, with a Yggdrasil Kotlin image.** Wire parity is the + library's promise. A runtime that strands acknowledged frames should not + stay listed as conforming. + - **Depends on it.** Cut 1 and Cut 3 scope and their verification host. + +Not forks, recorded so they are not re-asked: + +- `Acknowledged` is removed, not aliased. The ruled states replace it. +- Delivered means handed to the transport's application boundary, not consumed + (the ruling). +- The 4 MiB cap stays. The sender's window keeps legal traffic under it (§5). +- The receiver's 4,096 receive history stays. The sender's window keeps legal + traffic inside it. +- A receipt keeps Received across a session end or reset (the ruling). A reset + keeps one frozen epoch. + + +## Sequencing and blockers + +1. **`hands/rudp-stray-packets` merges to `main`.** It is in Hands now. Every + cut here is based on that merge. +2. **The Odin CultLib pin bump** follows that merge. It is **not** blocked on + this campaign: Odin crashed on stray packets, and it should not wait. Odin + uses `publish_cultnet_message_to_rudp_catalog`, not the enum, so Cut 4 does + not break its build when it bumps again. +3. **Cut 1** is blocked on step 1 only. +4. **Cut 2 and Cut 3** are blocked on step 1 only. They are independent of Cut 1 + and of each other, and can run in parallel worktrees. Cut 3 and Cut 1 touch + different functions in the same files. Merge Cut 1 first so that Cut 3's P4 + test exercises the new hold path. +5. **Cut 4** is blocked on Cut 1, because Delivered is sound only against a + correct receiver. It is also blocked on Q-A1 and Q-A3. +6. **Cut 5** is blocked on Cut 3 in TypeScript, because the socket transport it + routes through must carry the window. +7. **Parallel campaigns:** + - **`hands/media-fec-cut3`** (Muninn's Cut 3 in CultLib) touches `rudp.rs` + only in the profile's `audio` entry (`~2215`) and the + `"media" | "audio"` arm of `channel_send_options` (`~2380`). Neither + overlaps Cuts 1-4 (the session internals and `send_reliable` + `:1891-1928`). It can merge in either order, and the second to land + rebases trivially. Once it lands, reliable expiry is only for `media` when + the caller chooses Reliable. + - **Muninn's own cuts** use lossy media plus parity. Its reliable `audio` + session has no ordered traffic, so P2 does not bite it. Its next CultLib + pin (the `pins/` commit at Cut 3's CultLib tag, Muninn Q2) picks up + whatever has landed. Nothing here blocks Muninn, and nothing in Muninn + blocks this. + - **CultCache publication Cut 4** (CultNetDatabase on the journal, + `docs/cultcache-publication-cut.md` §4) touches `CultNetDatabase.cs` + only, and needs in-order, exactly-once delivery on `schema`. The C# + socket transports carry one reliable channel (`CultNetTransport.cs:2372-2376`), + so P1 and P6 cannot occur on their sessions. There is no blocking + relation in either direction. +8. **Heimdall and StreamPixels vendored bumps** (`b6b1d9c`, `542ddd3`) come + after Cut 5, so that they pick up honest flushes and the TS window at once. + Heimdall's `idunn-rudp-health.ts` first-ack wait moves to `flush` at that + bump. +9. **CodexConnector (`e171eca`) and Ghostlight (`85f7024`)** change + `Acknowledged` to `Delivered` (or to `wait_delivered`) at their next bump + after Cut 4. + + +## Follow-ups outside this campaign + +- **Hold caps in C#, TS, Python and Kotlin.** Their ordered hold buffers and + per-channel maps are unbounded against a hostile peer. Rust has caps. +- **The Rust server-mode socket transport resets on every Connect** + (`rudp.rs:2099`), even a repeated one. This is a stray-packet parity item. +- **The document server's per-session payload budget never shrinks** + (`rudp_document_server.rs:321`, `:584`). After 4 MiB in one session it drops + data without acknowledging it, and the sender retransmits until it gives up. +- **Fire-and-forget health.** Muninn `main.rs:9446`, Sleipnir `main.rs:880-904` + and Heimdall's first-ack wait can lose a health record silently. +- **FORWARD-TSN** if Q-A3 is A and a consumer later mixes expiry with ordered + traffic. +- **The P4 probe at the pins** predates its fix, so it proves nothing there. If + anyone needs old-sender-to-new-receiver evidence for the cap, rerun + `09318d13`'s P4 against a pinned sender. From ecfb4d65095a1df131938cb70cc884d70b40efca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:29:59 +0200 Subject: [PATCH 57/64] Ack map: record Q-A1 and Q-A2, one meaning for Acknowledged Co-Authored-By: Claude Opus 5.5 --- docs/cultnet-rudp-ack-cut.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md index f655e90a..e595a005 100644 --- a/docs/cultnet-rudp-ack-cut.md +++ b/docs/cultnet-rudp-ack-cut.md @@ -1,8 +1,20 @@ -# CultNet RUDP: a frame is Received, then Delivered, and the receiver owns ordered delivery with one watermark +# CultNet RUDP: Acknowledged means delivered, and the receiver owns ordered delivery with one watermark -Status: cut map, Imagination pass 1, 2026-09-30. It is not committed yet. Self -commits it to `main` as `docs/cultnet-rudp-ack-cut.md`. The ends are the operator -ruling of 2026-09-30, summarised below. This document owns the means. +Status: cut map, Imagination pass 1, 2026-09-30, committed by Self. + +**Rulings (operator, 2026-09-30).** These supersede the two-state proposal in the text below. Cuts 4 and 5 +need re-mapping to them before Hands. +- **Q-A1 (a):** Delivered is derived at the sender from a correct receiver's cumulative watermark. The wire + does not change. +- **Q-A2:** "I think this is telling us we don't actually need both received and delivered." + - There is one public meaning: Acknowledged means delivered. + - Receipts go Pending, then Acknowledged (delivered), or Invalidated. No public Received state exists. + - Retransmission still stops on each packet's own ack, so resending stays bounded. + - Flush waits for Acknowledged. +- **Open:** Q-A3 (expiring reliable traffic next to ordered traffic), Q-A4 (porting receipts), Q-A5 (Kotlin). +- **Added to Cut 1's scope** (from stray-packet Soul pass 3, pre-existing): a restarted client on the same + endpoint and payload is locked out by repeated-Connect answers. A reconnect on the same transport treats a new + server session's frames as duplicates. Session generation belongs to the sequence space this cut owns. - **Body.** `origin/hands/rudp-stray-packets` at `d85877a7` ("the session no longer keeps an initial sequence nothing reads"). Every `file:line` is against From 756da4894393d1b2ba761263dd715276f5a01a02 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:32:05 +0200 Subject: [PATCH 58/64] cultnet-ts: a Connect from another endpoint replaces the peer; a flush is bound to its session; a busy operation session is not idle A server-mode transport treated a Connect from a new endpoint as a repeated Connect once the session was connected: the new client got an Ack and never an Accept, and the old session never ended. Only a Connect from the accepted endpoint repeats; another endpoint resets the session and gets its Accept. A flush recorded nothing about the session it started in, so a disconnect listener that reconnected cleared the end reason before the 5 ms poll saw it. Sessions now carry a generation; a flush fails if its generation ended. The operation service's idle sweep deleted a session whose handler was still running, losing the response. A session with a handler running, or one that finished a handler within the idle window, is not idle. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/operation-service.ts | 16 ++- packages/cultnet-ts/src/rudp.ts | 40 +++++-- packages/cultnet-ts/test/cultnet.test.ts | 119 +++++++++++++++++++ 3 files changed, 162 insertions(+), 13 deletions(-) diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 433c762d..606cd418 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -51,6 +51,10 @@ export interface CultNetOperationClientOptions { interface RemoteSession { session: CultNetRudpSession; remote: RemoteInfo; + /** Handlers running for this session; a session serving a request is not idle. */ + handling: number; + /** When the last handler finished: the response still needs its acknowledgement. */ + lastHandledAtMs: number; } export async function startCultNetOperationServer( @@ -82,6 +86,7 @@ export async function startCultNetOperationServer( for (const [key, peer] of sessions) { // An abandoned client must not hold a slot forever, or the cap locks // every new client out. + if (peer.handling > 0 || now - peer.lastHandledAtMs < idleTimeoutMs) continue; if (peer.session.checkTimeout(now, idleTimeoutMs)) sessions.delete(key); } for (const peer of sessions.values()) { @@ -162,7 +167,7 @@ async function handleServerDatagram( if (packet.packetType === "connect") { if (!peer) { if (sessions.size >= MAX_OPERATION_SESSIONS) return false; - peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote }; + peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote, handling: 0, lastHandledAtMs: 0 }; sessions.set(key, peer); } // A Connect from an admitted peer repeats: the session answers with the @@ -189,7 +194,14 @@ async function handleServerDatagram( try { const message = parseCultNetMessage(decode(frame.payload)); if (message.schemaVersion !== "cultnet.operation_request.v0") continue; - const response = await options.handler(message); + peer.handling += 1; + let response: CultNetOperationResponseMessage; + try { + response = await options.handler(message); + } finally { + peer.handling -= 1; + peer.lastHandledAtMs = Date.now(); + } const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); for (const responsePacket of peer.session.sendMany("schema", payload, { reliable: true, diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 1dc4f032..7e99afe9 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -824,7 +824,12 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; - /** Why the session ended, while it is over; cleared by the next Connect. */ + // A flush belongs to the session it started in. Each session start advances + // the generation; each session end records the generation it ended and why. + // A reconnect from a disconnect listener starts a new generation but cannot + // un-end the old one. + #generation = 0; + #endedGeneration = -1; #endedReason: Uint8Array | undefined; readonly #stats: CultNetTransportStats = { bytesReceived: 0, @@ -887,7 +892,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement if (this.#mode !== "client") { throw new Error("Only a client RUDP socket transport can initiate connect."); } - this.#endedReason = undefined; + this.#generation += 1; this.#sendPacket(this.#session.createConnect(Date.now(), payload)); } @@ -905,12 +910,13 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement async flush(timeoutMs = 30_000): Promise { const deadline = Date.now() + Math.max(0, timeoutMs); + const generation = this.#generation; for (;;) { // An ended session forgot its unacknowledged writes; reporting them - // flushed would be a lie. - if (this.#endedReason !== undefined) { + // flushed would be a lie, even if a new session has since begun. + if (this.#endedGeneration >= generation) { throw new Error( - `RUDP session ended before its reliable writes were acknowledged: ${Buffer.from(this.#endedReason).toString("utf8")}`, + `RUDP session ended before its reliable writes were acknowledged: ${Buffer.from(this.#endedReason ?? new Uint8Array()).toString("utf8")}`, ); } if (this.#session.outstandingReliablePacketCount === 0) { @@ -928,6 +934,11 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } } + #endGeneration(reason: Uint8Array): void { + this.#endedGeneration = this.#generation; + this.#endedReason = reason; + } + ping(payload = new Uint8Array()): void { this.#sendPacket(this.#session.createPing(payload)); } @@ -961,7 +972,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); - this.#endedReason = reason; + this.#endGeneration(reason); try { this.#sendPacket(this.#session.createDisconnect(reason)); } catch { @@ -996,6 +1007,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } + let movedEndpoint = false; if (!this.#remoteHost || this.#remotePort === undefined) { if (this.#mode === "server" && packet.packetType !== "connect") { this.#stats.packetsDropped += 1; @@ -1005,6 +1017,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#remotePort = remote.port; } else if (remote.address !== this.#remoteHost || remote.port !== this.#remotePort) { if (this.#mode === "server" && packet.packetType === "connect") { + movedEndpoint = true; this.#remoteHost = remote.address; this.#remotePort = remote.port; } else { @@ -1014,15 +1027,20 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } if (this.#mode === "server" && packet.packetType === "connect") { - // A repeated Connect from the accepted peer resets nothing: the session - // answers it with the Accept already owed. - if (!this.#session.connected) { + // Only a Connect from the endpoint already accepted repeats, and the + // session answers it with the Accept owed. A Connect from another + // endpoint replaces the peer: the old session's writes die with it. + const repeated = this.#session.connected && !movedEndpoint; + if (!repeated) { + if (this.#session.connected) { + this.#endGeneration(Buffer.from("replaced by a new Connect", "utf8")); + } this.#session.resetPeerState(); } let accept: CultNetRudpPacket; try { accept = this.#session.acceptConnect(packet, Date.now()); - this.#endedReason = undefined; + if (!repeated) this.#generation += 1; } catch { this.#stats.packetsDropped += 1; return; @@ -1060,7 +1078,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } satisfies CultNetTransportFrame); } if (result.disconnected) { - this.#endedReason = result.disconnectReason ?? new Uint8Array(); + this.#endGeneration(result.disconnectReason ?? new Uint8Array()); this.emit("disconnect", { reason: result.disconnectReason ?? new Uint8Array() }); this.emit("close"); return; diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index cf4af8c0..bdaa9509 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2220,3 +2220,122 @@ test("operation service frees the slots of abandoned sessions once they idle out await server.close(); } }); + +test("server-mode transport replaces its peer when a Connect arrives from another endpoint", async () => { + const serverSocket = await bindUdpSocket(); + const socketA = await bindUdpSocket(); + const socketB = await bindUdpSocket(); + const connectionId = 0x10203058; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + const receivedByB: CultNetRudpPacket[] = []; + socketB.on("message", (wire) => receivedByB.push(decodeRudpPacket(wire))); + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + const peerA = new CultNetRudpSession({ connectionId }); + const peerB = new CultNetRudpSession({ connectionId }); + const send = (socket: Socket, packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + try { + send(socketA, peerA.createConnect(0)); + await waitFor(() => server.connected, "A accepted"); + send(socketB, peerB.createConnect(0)); + await waitFor(() => receivedByB.some((p) => p.packetType === "accept"), "B's Accept"); + peerB.receive(receivedByB.find((p) => p.packetType === "accept")!, 0); + + // B owns the session; A's traffic is a stranger's and is dropped. + const [fromA] = peerA.sendMany("schema", Buffer.from("from-A"), { reliable: true, ordered: true, nowMs: 0 }); + send(socketA, fromA!); + const [fromB] = peerB.sendMany("schema", Buffer.from("from-B"), { reliable: true, ordered: true, nowMs: 0 }); + send(socketB, fromB!); + await waitFor(() => frames.length > 0, "B's frame"); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.deepEqual(frames, ["from-B"]); + assert.equal(server.stats.packetsDropped, 1); + } finally { + socketA.close(); + socketB.close(); + assert.doesNotThrow(() => server.close()); + } +}); + +test("operation service does not idle out a session while its handler is running", async () => { + const respond = (request: CultNetOperationRequestMessage): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }); + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + sessionIdleTimeoutMs: 200, + handler: async (request) => { + await new Promise((resolve) => setTimeout(resolve, 700)); + return respond(request); + }, + }); + try { + const response = await invokeCultNetOperation(server.endpoint, { + schemaVersion: "cultnet.operation_request.v0", + messageId: "slow", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }, { runtimeId: "eve-test", timeoutMs: 3000 }); + assert.equal(response.messageId, "slow"); + } finally { + await server.close(); + } +}); + +test("a flush is bound to the session it started in, even when a disconnect listener reconnects", async () => { + const peerSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203059; + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(peerSocket), + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + let clientPort = 0; + peerSocket.on("message", (wire, remote) => { + clientPort = remote.port; + received.push(decodeRudpPacket(wire)); + }); + const peer = new CultNetRudpSession({ connectionId }); + try { + client.connect(); + await waitFor(() => received.some((p) => p.packetType === "connect"), "the Connect"); + const accept = peer.acceptConnect(received.find((p) => p.packetType === "connect")!, 0); + peerSocket.send(encodeRudpPacket(accept), clientPort, "127.0.0.1"); + await waitFor(() => client.connected, "the client connected"); + client.send("schema", Buffer.from("never acknowledged")); + client.on("disconnect", () => client.connect()); + const flushed = client.flush(500); + const outcome = assert.rejects(flushed, /ended before its reliable writes were acknowledged/); + peerSocket.send(encodeRudpPacket(peer.createDisconnect(Buffer.from("bye"))), clientPort, "127.0.0.1"); + await outcome; + } finally { + peerSocket.close(); + client.close(); + } +}); From 5f1e883d884ba6847d1f45d1a1b167e6d557e92d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:35:44 +0200 Subject: [PATCH 59/64] cultnet-rs: a peer's Disconnect ends its session's receipts and waits for good A Disconnect left the session's pending writes and scope alone, and the transport's Invalidated status leaned on disconnect_reason, which connect() clears. After a reconnect the receipt returned to Pending and could become Acknowledged in the new session. The session now rotates its scope and drops what it still owed on Disconnect, so the session is the one owner of Invalidated and the transport override is gone. A flush records the session scope it started in and fails when a Connect that replaces the peer rotates it, instead of resolving on the emptied queue. Tests: Disconnect-then-reconnect receipt, stale end reason after a new Connect is accepted, flush across a replaced peer, and a repeated Connect's ack field. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-rs/src/rudp.rs | 28 ++-- .../cultnet-rs/tests/rudp_stray_packets.rs | 129 ++++++++++++++++++ 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 29467890..5f7084ce 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -126,8 +126,8 @@ pub struct CultNetRudpSendOptions { } /// Identifies every transport packet belonging to one non-expiring reliable -/// send. Receipts are local to the session that issued them and cannot survive -/// a peer reset. +/// send. Receipts are local to the session that issued them: a peer reset or +/// a Disconnect invalidates them for good. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpReliableSendReceipt { session_scope: Uuid, @@ -278,6 +278,10 @@ impl CultNetRudpSession { Some(pending.packet.clone()) } + fn session_scope(&self) -> Uuid { + self.session_scope + } + fn reliable_send_status( &self, receipt: &CultNetRudpReliableSendReceipt, @@ -588,7 +592,13 @@ impl CultNetRudpSession { } if packet.packet_type == CultNetRudpPacketType::Disconnect { + // The session is over. What it still owed the peer dies with it, + // and receipts it issued can never become Acknowledged in a later + // session. self.connected = false; + self.session_scope = Uuid::new_v4(); + self.pending_reliable.clear(); + self.queued_reliable.clear(); return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), ready_to_send, @@ -1920,12 +1930,7 @@ impl CultNetRudpSocketTransportConnection { &self, receipt: &CultNetRudpReliableSendReceipt, ) -> CultNetRudpReliableSendStatus { - let status = self.session.reliable_send_status(receipt); - // A write still unacknowledged when the session ended never will be. - if status == CultNetRudpReliableSendStatus::Pending && self.disconnect_reason.is_some() { - return CultNetRudpReliableSendStatus::Invalidated; - } - status + self.session.reliable_send_status(receipt) } fn send_data_packets(&mut self, packets: &[CultNetRudpPacket]) -> Result<()> { @@ -1990,15 +1995,18 @@ impl CultNetRudpSocketTransportConnection { self.socket.set_read_timeout(Some(poll_timeout))?; let deadline = Instant::now() + timeout; + // A flush belongs to the session it started in: a Connect that + // replaces the peer mid-wait forgets the writes being waited on. + let scope = self.session.session_scope(); let mut preserved_frames = VecDeque::new(); let result = (|| { loop { // An ended session forgot its unacknowledged writes; reporting // them flushed would be a lie. - if let Some(reason) = &self.disconnect_reason { + if self.session.session_scope() != scope || self.disconnect_reason.is_some() { return Err(anyhow!( "RUDP session ended before its reliable writes were acknowledged: {}", - String::from_utf8_lossy(reason) + String::from_utf8_lossy(self.disconnect_reason.as_deref().unwrap_or(b"replaced by a new Connect")) )); } if self.outstanding_reliable_packet_count() == 0 { diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index bbb96395..25dd80e6 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -620,3 +620,132 @@ fn a_write_unacknowledged_when_the_peer_disconnects_fails_its_flush() -> Result< ); Ok(()) } + +/// Drains what a raw peer socket has been sent. +fn drain(socket: &UdpSocket) -> Result> { + let mut buffer = vec![0_u8; 65_535]; + let mut packets = Vec::new(); + while let Ok((received, _)) = socket.recv_from(&mut buffer) { + packets.push(decode_rudp_packet(&buffer[..received])?); + } + Ok(packets) +} + +/// A receipt belongs to the session it was issued in. A peer's Disconnect ends +/// that session for good: reconnecting must not bring the write back to Pending, +/// and an ack the old peer sent late must not make it Acknowledged. +#[test] +fn a_receipt_invalidated_by_a_peer_disconnect_never_becomes_acknowledged() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server.recv_from(&mut buffer)?; + let mut old_peer = raw_session(CONNECTION_ID); + let accept = old_peer.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + let receipt = client.send_reliable("schema", b"written before the disconnect".to_vec())?; + let data = drain(&server)? + .into_iter() + .find(|packet| packet.packet_type == CultNetRudpPacketType::Data) + .expect("the write reached the peer"); + old_peer.receive(&data, 0)?; + let late_ack = old_peer.create_ack_for_received(data.sequence); + send_to(&server, client_addr, &old_peer.create_disconnect(b"bye".to_vec()))?; + let _ = client.receive_once()?; + assert_eq!(client.reliable_send_status(&receipt), CultNetRudpReliableSendStatus::Invalidated); + + client.connect(b"hello again".to_vec())?; + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated, + "reconnecting brought the write back to Pending" + ); + let connect = drain(&server)? + .into_iter() + .find(|packet| packet.packet_type == CultNetRudpPacketType::Connect) + .expect("the reconnect reached the peer"); + let mut new_peer = raw_session(CONNECTION_ID); + let new_accept = new_peer.accept_connect(&connect, 0, Vec::new())?; + send_to(&server, client_addr, &new_accept)?; + send_to(&server, client_addr, &late_ack)?; + while client.receive_once()?.is_some() {} + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated, + "the old session's write was acknowledged in the new session" + ); + // The old write is not retransmitted into the new session either. + assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the reconnect's own Connect is outstanding"); + Ok(()) +} + +/// A server-mode transport that accepts a new Connect has a live session again; +/// the reason the previous one ended is history, not state. +#[test] +fn server_mode_forgets_the_end_reason_once_a_new_connect_is_accepted() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut peer_session = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &peer_session.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + send_to(&peer, server_addr, &peer_session.create_disconnect(b"bye".to_vec()))?; + let _ = server.receive_once()?; + assert_eq!(server.disconnect_reason(), Some(&b"bye"[..])); + + let mut next_peer = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &next_peer.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + assert!(server.connected()); + assert_eq!(server.disconnect_reason(), None, "a stale reason survives the new session"); + Ok(()) +} + +/// A flush waits on one session. A Connect from another endpoint replaces the +/// peer and forgets the writes being waited on; the flush must say so. +#[test] +fn a_flush_fails_when_a_new_endpoint_replaces_the_peer_mid_wait() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer_a = socket()?; + let peer_b = socket()?; + send_to(&peer_a, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"a".to_vec())?)?; + let _ = server.receive_once()?; + server.send_reliable("schema", b"for A".to_vec())?; + send_to(&peer_b, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"b".to_vec())?)?; + + let error = server.flush_reliable(Duration::from_millis(300)).expect_err("A's write was forgotten"); + assert!( + error.to_string().contains("ended before its reliable writes were acknowledged"), + "{error}" + ); + Ok(()) +} + +/// A repeated Connect still carries an acknowledgement field, and the session +/// honours it: the Accept it acknowledges is no longer owed. +#[test] +fn a_repeated_connect_acknowledges_what_it_carries() -> Result<()> { + let mut client = raw_session(CONNECTION_ID); + let mut server = raw_session(CONNECTION_ID); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + let mut repeat = connect.clone(); + repeat.ack = accept.sequence; + let reply = server.answer_repeated_connect(&repeat, 1)?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + assert_eq!(server.outstanding_reliable_packet_count(), 0); + Ok(()) +} From 29f8bf575eb9822e64584d14a0859b77aada8bb3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:35:44 +0200 Subject: [PATCH 60/64] Tests: a repeated Connect acknowledges what it carries (TS, Python, C#); fix the TS replaced-peer test The repeated-Connect answer applies its acknowledgement field in every runtime; no test failed when a runtime stopped doing so. The TS replaced-peer test now lets client A receive its Accept before A sends. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-py/tests/test_cultnet.py | 9 +++++++++ packages/cultnet-ts/test/cultnet.test.ts | 16 +++++++++++++++- .../GameCult.Networking.Tests/NetworkingTests.cs | 14 ++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 57c24421..7e78c48f 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -695,6 +695,15 @@ class CultNetTests(unittest.TestCase): self.assertTrue(client.check_timeout(91, 50)) self.assertFalse(client.connected) + def test_a_repeated_connect_acknowledges_what_it_carries(self) -> None: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=100)) + connect = client.create_connect(0, b"join") + accept = server.accept_connect(connect, 0) + reply = server.answer_repeated_connect(replace(connect, ack=accept.sequence), 1) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + self.assertEqual(server.outstanding_reliable_packet_count, 0) + def test_cultnet_rudp_session_bounds_pending_reliable_packets_before_enqueue(self) -> None: session = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=102, initial_sequence=1, max_pending_reliable_packets=2) diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index bdaa9509..093b49d6 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2233,6 +2233,8 @@ test("server-mode transport replaces its peer when a Connect arrives from anothe connectionId, resendPollMs: 1000, }); + const receivedByA: CultNetRudpPacket[] = []; + socketA.on("message", (wire) => receivedByA.push(decodeRudpPacket(wire))); const receivedByB: CultNetRudpPacket[] = []; socketB.on("message", (wire) => receivedByB.push(decodeRudpPacket(wire))); const frames: string[] = []; @@ -2243,7 +2245,8 @@ test("server-mode transport replaces its peer when a Connect arrives from anothe socket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); try { send(socketA, peerA.createConnect(0)); - await waitFor(() => server.connected, "A accepted"); + await waitFor(() => receivedByA.some((p) => p.packetType === "accept"), "A's Accept"); + peerA.receive(receivedByA.find((p) => p.packetType === "accept")!, 0); send(socketB, peerB.createConnect(0)); await waitFor(() => receivedByB.some((p) => p.packetType === "accept"), "B's Accept"); peerB.receive(receivedByB.find((p) => p.packetType === "accept")!, 0); @@ -2339,3 +2342,14 @@ test("a flush is bound to the session it started in, even when a disconnect list client.close(); } }); + +test("a repeated Connect acknowledges what it carries", () => { + const connectionId = 0x1020305a; + const client = new CultNetRudpSession({ connectionId }); + const server = new CultNetRudpSession({ connectionId }); + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 0); + const reply = server.answerRepeatedConnect({ ...connect, ack: accept.sequence }, 1); + assert.equal(reply.packetType, "ack"); + assert.equal(server.outstandingReliablePacketCount, 0); +}); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index ce39ea30..e32ca91a 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -900,6 +900,20 @@ namespace GameCult.Networking.Tests Assert.That(client.Connected, Is.False); } + [Test] + public void RudpSession_RepeatedConnectAcknowledgesWhatItCarries() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 103, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 103, InitialSequence = 100 }); + var connect = client.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + var accept = server.AcceptConnect(connect, 0); + var repeat = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(connect)); + repeat.Ack = accept.Sequence; + var reply = server.AnswerRepeatedConnect(repeat, 1); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(0)); + } + [Test] public void RudpSession_LossyPacketsCannotCreateReliableOrderedGaps() { From c412eed0667fb20e95d1b6a88f7036dc7381a4f8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:37:24 +0200 Subject: [PATCH 61/64] cultnet-rs: the reconnect test expects the Accept to have acknowledged the new Connect Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-rs/tests/rudp_stray_packets.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 25dd80e6..33f02794 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -680,8 +680,9 @@ fn a_receipt_invalidated_by_a_peer_disconnect_never_becomes_acknowledged() -> Re CultNetRudpReliableSendStatus::Invalidated, "the old session's write was acknowledged in the new session" ); - // The old write is not retransmitted into the new session either. - assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the reconnect's own Connect is outstanding"); + // The old write is not retransmitted into the new session either: the + // Accept acknowledged the reconnect's Connect, and nothing else is owed. + assert_eq!(client.outstanding_reliable_packet_count(), 0, "the old write survived into the new session"); Ok(()) } From 36e052b33ae6b0bb26a737c117c186a0c9682046 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:46:27 +0200 Subject: [PATCH 62/64] cultnet-ts: the replaced-peer test acknowledges A's Accept so a repeated-Connect answer is distinguishable from an Accept Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/test/cultnet.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 093b49d6..5ba9fc05 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2246,7 +2246,10 @@ test("server-mode transport replaces its peer when a Connect arrives from anothe try { send(socketA, peerA.createConnect(0)); await waitFor(() => receivedByA.some((p) => p.packetType === "accept"), "A's Accept"); - peerA.receive(receivedByA.find((p) => p.packetType === "accept")!, 0); + const acceptA = receivedByA.find((p) => p.packetType === "accept")!; + peerA.receive(acceptA, 0); + send(socketA, peerA.createAckForReceived(acceptA.sequence)); + await waitFor(() => server.outstandingReliablePacketCount === 0, "A's Accept acknowledged"); send(socketB, peerB.createConnect(0)); await waitFor(() => receivedByB.some((p) => p.packetType === "accept"), "B's Accept"); peerB.receive(receivedByB.find((p) => p.packetType === "accept")!, 0); From d29e2a7d81b8ddb4a8b6269aabf2fc006944196f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:59:18 +0200 Subject: [PATCH 63/64] Ack map: the ending paths disagree about owed writes (Cut 1 scope) Co-Authored-By: Claude Opus 5.5 --- docs/cultnet-rudp-ack-cut.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md index e595a005..f21b2006 100644 --- a/docs/cultnet-rudp-ack-cut.md +++ b/docs/cultnet-rudp-ack-cut.md @@ -15,6 +15,12 @@ need re-mapping to them before Hands. - **Added to Cut 1's scope** (from stray-packet Soul pass 3, pre-existing): a restarted client on the same endpoint and payload is locked out by repeated-Connect answers. A reconnect on the same transport treats a new server session's frames as duplicates. Session generation belongs to the sequence space this cut owns. +- **Also in Cut 1's scope** (stray-packet Soul pass 4): the ending paths disagree about what a session owed. + - In Rust, a peer's Disconnect invalidates receipts. + - A local `disconnect()` or a timeout carries owed writes into the next session. There, a receipt from the old + session can become Acknowledged (probe R1). + - TS, Python and C# also carry owed writes across a peer Disconnect. + - One rule: a write belongs to the session it was issued in. - **Body.** `origin/hands/rudp-stray-packets` at `d85877a7` ("the session no longer keeps an initial sequence nothing reads"). Every `file:line` is against From b0d4103e22b5fd279b00bc1829d61655caffc1b8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 22:01:43 +0200 Subject: [PATCH 64/64] cultnet-ts: a timeout ends the session generation, and every non-repeated Connect ends a live one Also adds a swept-after-handler test for the operation service and states the Rust receipt comment exactly: only a peer Disconnect invalidates receipts. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-rs/src/rudp.rs | 4 +- packages/cultnet-ts/src/rudp.ts | 5 +- packages/cultnet-ts/test/cultnet.test.ts | 134 +++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 5f7084ce..3b348fe3 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -127,7 +127,9 @@ pub struct CultNetRudpSendOptions { /// Identifies every transport packet belonging to one non-expiring reliable /// send. Receipts are local to the session that issued them: a peer reset or -/// a Disconnect invalidates them for good. +/// a Disconnect received from the peer invalidates them for good. A local +/// `disconnect()` or a timeout does not; the writes it leaves owed carry into +/// the next session. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpReliableSendReceipt { session_scope: Uuid, diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 7e99afe9..ae7ea838 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -946,6 +946,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement checkTimeout(timeoutMs: number, nowMs = Date.now()): boolean { const timedOut = this.#session.checkTimeout(nowMs, timeoutMs); if (timedOut) { + this.#endGeneration(Buffer.from("session timed out", "utf8")); this.emit("timeout"); this.emit("close"); } @@ -1032,7 +1033,9 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // endpoint replaces the peer: the old session's writes die with it. const repeated = this.#session.connected && !movedEndpoint; if (!repeated) { - if (this.#session.connected) { + // A timed-out session already ended its generation; any other + // predecessor ends here, connected or not. + if (this.#endedGeneration < this.#generation) { this.#endGeneration(Buffer.from("replaced by a new Connect", "utf8")); } this.#session.resetPeerState(); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 5ba9fc05..35cba767 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -8,6 +8,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { z } from "zod"; +import { encode } from "@msgpack/msgpack"; import { CultCache, SingleFileMessagePackBackingStore, @@ -2356,3 +2357,136 @@ test("a repeated Connect acknowledges what it carries", () => { assert.equal(reply.packetType, "ack"); assert.equal(server.outstandingReliablePacketCount, 0); }); + +async function timedOutServerWithUnackedWrite(connectionId: number) { + const serverSocket = await bindUdpSocket(); + const socketA = await bindUdpSocket(); + const socketB = await bindUdpSocket(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const receivedByA: CultNetRudpPacket[] = []; + socketA.on("message", (wire) => receivedByA.push(decodeRudpPacket(wire))); + const peerA = new CultNetRudpSession({ connectionId }); + const send = (socket: Socket, packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + send(socketA, peerA.createConnect(0)); + await waitFor(() => receivedByA.some((p) => p.packetType === "accept"), "A's Accept"); + const acceptA = receivedByA.find((p) => p.packetType === "accept")!; + peerA.receive(acceptA, 0); + send(socketA, peerA.createAckForReceived(acceptA.sequence)); + await waitFor(() => server.outstandingReliablePacketCount === 0, "A's Accept acknowledged"); + server.send("schema", Buffer.from("never acknowledged")); + assert.ok(server.outstandingReliablePacketCount > 0); + const flushed = server.flush(2000); + const outcome = assert.rejects(flushed, /ended before its reliable writes were acknowledged/); + // The flush must have started before the timeout fires. + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(server.checkTimeout(50, Date.now() + 10_000), true); + return { + server, socketA, socketB, outcome, send, + close: () => { + socketA.close(); + socketB.close(); + server.close(); + }, + }; +} + +test("a flush waiting on an unacknowledged write fails as soon as the session times out", async () => { + const t = await timedOutServerWithUnackedWrite(0x1020305a); + try { + await t.outcome; + } finally { + t.close(); + } +}); + +test("a flush fails when a Connect from another endpoint follows a timeout", async () => { + const t = await timedOutServerWithUnackedWrite(0x1020305b); + try { + t.send(t.socketB, new CultNetRudpSession({ connectionId: 0x1020305b }).createConnect(0)); + await t.outcome; + } finally { + t.close(); + } +}); + +test("a flush fails when a Connect from the same endpoint follows a timeout", async () => { + const t = await timedOutServerWithUnackedWrite(0x1020305c); + try { + t.send(t.socketA, new CultNetRudpSession({ connectionId: 0x1020305c }).createConnect(0)); + await t.outcome; + } finally { + t.close(); + } +}); + +test("operation service sweeps a session that ran a handler once it idles out", async () => { + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + sessionIdleTimeoutMs: 300, + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }), + }); + const port = Number(new URL(server.endpoint).port); + const sockets: Socket[] = []; + const request: CultNetOperationRequestMessage = { + schemaVersion: "cultnet.operation_request.v0", + messageId: "after-handling", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }; + const connectionId = 0x43554c54; + try { + // Sixty-four clients each run a handler and then go silent without a + // Disconnect. Only the idle sweep can free their slots. + const clients: { socket: Socket; peer: CultNetRudpSession; handled: () => boolean }[] = []; + for (let index = 0; index < 64; index += 1) { + const socket = await bindUdpSocket(); + sockets.push(socket); + const peer = new CultNetRudpSession({ connectionId }); + let handled = false; + let accepted = false; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "accept" && !accepted) { + accepted = true; + peer.receive(packet, 0); + for (const out of peer.sendMany("schema", Buffer.from(encode(encodeCultNetMessageForWire(request, "cultnet.schema.v0"))), { reliable: true, ordered: true, nowMs: 0 })) { + socket.send(encodeRudpPacket(out), port, "127.0.0.1"); + } + } else if (packet.channelId === "schema") { + handled = true; + } + }); + socket.send(encodeRudpPacket(peer.createConnect(0)), port, "127.0.0.1"); + clients.push({ socket, peer, handled: () => handled }); + } + await waitFor(() => clients.every((c) => c.handled()), "every handler ran"); + await new Promise((resolve) => setTimeout(resolve, 700)); + const response = await invokeCultNetOperation(server.endpoint, request, { runtimeId: "eve-test", timeoutMs: 1000 }); + assert.equal(response.messageId, "after-handling"); + } finally { + for (const socket of sockets) socket.close(); + await server.close(); + } +});