CultLib/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs
Meta Vens 3b9ffac2fe C#: a send failure's goodbye belongs to the generation that sent
Send ended an unsendable session after releasing SessionGate, and the
goodbye was built under the gate but sent outside it. If the polling
thread admitted a new Connect from the same endpoint in that window, the
ended generation's goodbye reached the new client: it saw Data, Accept,
Disconnect and believed itself disconnected while the listener held it
connected.

Every path that finds a permanent send failure now reads the session's
generation under the gate at the send, and EndUnsendableSession builds
and sends the goodbye under the gate only while that generation is
unchanged. The generation check replaces the UnsendableReason once-guard:
a second failure in the same generation finds it already ended.

The test orders the new Connect inside the send, before the fragment
that can never be sent, and asserts the new client hears no Disconnect
and stays connected on both sides. It fails on the pre-fix code
(Data, Accept, Disconnect) and kills the mutant that drops the check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 17:23:30 +02:00

548 lines
26 KiB
C#

#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using NUnit.Framework;
namespace GameCult.Networking.Tests
{
/// <summary>
/// One peer whose datagrams cannot be sent must not stop the listener serving the others. The
/// failure is injected at the listener's one send function through <c>FailingSendPeers</c>.
/// </summary>
public class CultNetRudpSendFailureTests
{
private const uint ConnectionId = 0x10203099;
private sealed class Peer
{
public Socket Socket = null!;
public CultNetRudpSession Session = null!;
public CultNetRudpSocketServerPeer Server = null!;
public EndPoint EndPoint => Socket.LocalEndPoint!;
}
private static Socket Bind()
{
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
return socket;
}
private static List<CultNetRudpPacket> Drain(Socket socket)
{
var packets = new List<CultNetRudpPacket>();
var buffer = new byte[65535];
EndPoint from = new IPEndPoint(IPAddress.Any, 0);
while (socket.Poll(20_000, SelectMode.SelectRead))
{
var received = socket.ReceiveFrom(buffer, ref from);
packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray()));
}
return packets;
}
private static Peer Connect(CultNetRudpSocketTransportServer server, EndPoint serverEndPoint)
{
var peer = new Peer
{
Socket = Bind(),
Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId })
};
peer.Socket.SendTo(CultNetRudpPacketCodec.Encode(peer.Session.CreateConnect(0)), serverEndPoint);
server.ReceiveOnce();
peer.Session.Receive(Drain(peer.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
peer.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(peer.EndPoint));
return peer;
}
private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null, uint? initialSequence = null)
{
var serverSocket = Bind();
var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
{
RuntimeId = "csharp-send-failure",
Socket = serverSocket,
ConnectionId = ConnectionId,
ResendDelayMs = 5,
MaxFragmentBytes = maxFragmentBytes,
InitialSequence = initialSequence
});
var x = Connect(server, serverSocket.LocalEndPoint!);
var y = Connect(server, serverSocket.LocalEndPoint!);
return (server, serverSocket.LocalEndPoint!, x, y);
}
private static CultNetRudpPacket ReliableFrame(Peer peer, string text) =>
peer.Session.Send("schema", Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true });
[Test]
public void AFailedResendToOnePeerDoesNotStopTheOthersResends()
{
var (server, _, x, y) = TwoPeers();
using (server)
{
// Both peers' first datagram is lost, so both have a reliable resend due.
server.FailingSendPeers.Add(x.EndPoint);
server.FailingSendPeers.Add(y.EndPoint);
server.SendSchema(x.Server, "for-x");
server.SendSchema(y.Server, "for-y");
Assert.That(server.Stats.SendFailures, Is.EqualTo(2));
// Y's path recovers; X's stays dead. X is visited first.
server.FailingSendPeers.Remove(y.EndPoint);
Thread.Sleep(30);
Assert.DoesNotThrow(() => server.PollResends());
var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList();
Assert.That(resent, Has.Count.EqualTo(1));
Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y"));
Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(3));
Assert.That(server.Peers, Has.Count.EqualTo(2), "a send failure ends no session");
}
}
[Test]
public void AFailedAckToOnePeerDoesNotStopTheListenerServingTheOthers()
{
var (server, listenerEndPoint, x, y) = TwoPeers();
using (server)
{
server.FailingSendPeers.Add(x.EndPoint);
x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);
y.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(y, "from-y")), listenerEndPoint);
var frames = new List<string>();
for (var attempt = 0; attempt < 20 && frames.Count < 2; attempt++)
{
Assert.DoesNotThrow(() =>
{
if (server.ReceiveOnce() is { } frame)
frames.Add(Encoding.UTF8.GetString(frame.Frame.Payload));
});
}
Assert.That(frames, Is.EquivalentTo(new[] { "from-x", "from-y" }));
Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(1), "X's ack was counted as lost");
Assert.That(Drain(y.Socket).Select(p => p.PacketType), Does.Contain(CultNetRudpPacketType.Ack));
Assert.That(server.Peers, Has.Count.EqualTo(2));
}
}
// A permanent send failure is one that can never succeed for the datagram as built. A
// caller-directed send throws it and queues nothing; inside a poll it ends that peer's
// session and never the poll.
private static bool IsUnsendableReason(byte[]? reason)
{
var text = Encoding.UTF8.GetString(reason ?? Array.Empty<byte>());
return text.StartsWith("packet could not be sent: ", StringComparison.Ordinal)
&& text.Length > "packet could not be sent: ".Length;
}
private static List<CultNetRudpSocketServerPeer> RecordDisconnects(CultNetRudpSocketTransportServer server)
{
var ended = new List<CultNetRudpSocketServerPeer>();
server.PeerDisconnected += ended.Add;
return ended;
}
[Test]
public void AnOversizedSendThrowsTheErrorAndQueuesNothing()
{
var (server, _, x, y) = TwoPeers();
using (server)
{
// Each peer's own Accept is still awaiting its ack; the refused send adds nothing to it.
var outstanding = x.Server.Session.OutstandingReliablePacketCount;
var otherOutstanding = y.Server.Session.OutstandingReliablePacketCount;
var error = Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[70_000]))!;
Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString());
Assert.That(server.Stats.SendFailures, Is.Zero);
Assert.That(server.Stats.FramesSent, Is.Zero);
Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session");
Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing is pending or queued to resend");
// The refused send consumed no sequence: the next frame is delivered in order instead
// of waiting behind a gap that never fills.
server.SendSchema(x.Server, "next");
var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data);
var delivered = x.Session.Receive(data, 1).Delivered;
Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" }));
Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(otherOutstanding));
}
}
[Test]
public void ARefusedSendWithdrawsTheFragmentsTheWindowQueuedAsWell()
{
var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1);
using (server)
{
var outstanding = x.Server.Session.OutstandingReliablePacketCount;
server.UnsendableAfter[x.EndPoint] = 0;
// 1,100 one-byte fragments: the flow window admits the first ones and queues the rest,
// and the first datagram can never be sent.
Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[1_100]));
Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing of the refused send stays queued");
}
}
[Test]
public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession()
{
var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000);
using (server)
{
var ended = RecordDisconnects(server);
server.UnsendableAfter[x.EndPoint] = 1;
var error = Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[3000]))!;
Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True);
// The send ends the session and tells the peer; it leaves the peer for the polling
// thread to retire, because only that thread changes the peer set.
Assert.That(x.Server.Connected, Is.False);
Assert.That(Drain(x.Socket).Last().PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect));
Assert.That(ended, Is.Empty);
server.ReceiveOnce();
Assert.That(server.Peers, Has.Count.EqualTo(1));
Assert.That(ended, Is.EqualTo(new[] { x.Server }));
Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
}
}
[Test]
public void APeerEndedByASendOnAnotherThreadIsRetiredByThePollingThread()
{
var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000);
using (server)
{
var retiredOn = new List<int>();
server.PeerDisconnected += _ => retiredOn.Add(Environment.CurrentManagedThreadId);
server.UnsendableAfter[x.EndPoint] = 1;
SocketException? thrown = null;
var sender = new Thread(() =>
{
try { server.SendSchema(x.Server, new byte[3000]); }
catch (SocketException error) { thrown = error; }
});
sender.Start();
sender.Join();
Assert.That(thrown, Is.Not.Null);
Assert.That(server.Peers, Has.Count.EqualTo(2), "the sending thread leaves the peer set alone");
server.PollResends();
Assert.That(retiredOn, Is.EqualTo(new[] { Environment.CurrentManagedThreadId }));
Assert.That(server.Peers, Has.Count.EqualTo(1));
}
}
// A client that connects again from the same endpoint replaces the peer the listener held. A
// caller still holding the replaced peer (a handler still running for it, say) reaches nothing
// of the session that replaced it: no data and no goodbye.
/// <summary>X connects again as a new session, and the handshake completes.</summary>
private static (CultNetRudpSocketServerPeer Stale, CultNetRudpSocketServerPeer Replacing) Reconnect(
CultNetRudpSocketTransportServer server, EndPoint listenerEndPoint, Peer x)
{
var stale = x.Server;
Drain(x.Socket);
x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId });
x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint);
server.ReceiveOnce();
x.Session.Receive(Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
x.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint));
Assert.That(x.Server, Is.Not.SameAs(stale));
Assert.That(x.Session.Connected, Is.True);
return (stale, x.Server);
}
[Test]
public void ASendOnAReplacedPeerThrowsAndNoGoodbyeReachesTheSessionThatReplacedIt()
{
var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
using (server)
{
var (stale, replacing) = Reconnect(server, listenerEndPoint, x);
var ended = RecordDisconnects(server);
Assert.That(stale.Connected, Is.False, "the replaced peer's session ended with the replacement");
// Had the stale send gone out, a failure after its first fragment would end the
// session that sent it and say goodbye to the endpoint.
server.UnsendableAfter[x.EndPoint] = 1;
Assert.Throws<InvalidOperationException>(() => server.SendSchema(stale, new byte[3000]));
server.ReceiveOnce();
Assert.That(Drain(x.Socket), Is.Empty, "nothing of the stale send reached the client");
Assert.That(x.Session.Connected, Is.True);
Assert.That(replacing.Connected, Is.True);
Assert.That(server.Peers, Has.Member(replacing));
Assert.That(server.Peers, Has.Count.EqualTo(2));
Assert.That(ended, Is.Empty);
}
}
// With a fixed initial sequence the stale frame would take the sequence the real one gets, and
// the real one would be dropped as a duplicate. With a drawn one it would land far ahead of
// the new session's window, and nothing after it would be delivered.
[TestCase(1000u)]
[TestCase(null)]
public void ASendOnAReplacedPeerDeliversNothingAndTheReplacingPeersFrameArrives(uint? initialSequence)
{
var (server, listenerEndPoint, x, _) = TwoPeers(initialSequence: initialSequence);
using (server)
{
var (stale, replacing) = Reconnect(server, listenerEndPoint, x);
Assert.Throws<InvalidOperationException>(() => server.SendSchema(stale, "STALE"));
server.SendSchema(replacing, "REAL");
var delivered = Drain(x.Socket)
.SelectMany(packet => x.Session.Receive(packet, 1).Delivered)
.Select(frame => Encoding.UTF8.GetString(frame.Payload))
.ToList();
Assert.That(delivered, Is.EqualTo(new[] { "REAL" }));
}
}
// A send that fails after its first fragment ends the generation it began in, and only that
// one. Here X connects again while the send is under way, and the polling thread admits the
// new Connect before the fragment that can never be sent. The hook runs that poll inside the
// send so the order is fixed; in production it runs on the polling thread between the send
// and the failure's handling. The goodbye belonged to the ended generation, and the client
// must not hear it: it would believe itself disconnected while the listener holds it
// connected.
[Test]
public void ASendFailureAfterANewConnectReplacedThePeerSaysNoGoodbyeToTheNewSession()
{
var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
using (server)
{
var stale = x.Server;
Drain(x.Socket);
x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId });
x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint);
var fragments = 0;
server.BeforeSend = (remote, packet) =>
{
if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Data || ++fragments < 2)
return;
server.BeforeSend = null;
server.ReceiveOnce();
server.UnsendableAfter[x.EndPoint] = 0;
};
Assert.Throws<SocketException>(() => server.SendSchema(stale, new byte[3000]));
server.ReceiveOnce();
var packets = Drain(x.Socket);
Assert.That(packets.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect),
"the ended generation's goodbye reached the session that replaced it");
x.Session.Receive(packets.Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
Assert.That(x.Session.Connected, Is.True);
var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint));
Assert.That(replacing, Is.Not.SameAs(stale));
Assert.That(replacing.Connected, Is.True);
}
}
[Test]
public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce()
{
var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
using (server)
{
var ended = RecordDisconnects(server);
// The polling thread acknowledges X's frame outside the session's gate. Just before
// that ack goes out, a caller's send to X fails after its first fragment and ends the
// session; then the ack can never be sent either.
server.BeforeSend = (remote, packet) =>
{
if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Ack)
return;
server.BeforeSend = null;
server.UnsendableAfter[x.EndPoint] = 1;
var caller = new Thread(() =>
{
try { server.SendSchema(x.Server, new byte[3000]); }
catch (SocketException) { }
});
caller.Start();
caller.Join();
server.UnsendableAfter[x.EndPoint] = 0;
};
x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);
server.ReceiveOnce();
server.PollResends();
var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList();
Assert.That(goodbyes, Has.Count.EqualTo(1));
Assert.That(ended, Is.EqualTo(new[] { x.Server }));
Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
}
}
[Test]
public void APermanentFailureInResendsEndsOnlyThatPeersSession()
{
var (server, _, x, y) = TwoPeers();
using (server)
{
var ended = RecordDisconnects(server);
server.FailingSendPeers.Add(x.EndPoint);
server.FailingSendPeers.Add(y.EndPoint);
server.SendSchema(x.Server, "for-x");
server.SendSchema(y.Server, "for-y");
// X's datagram can now never be sent as built; Y's path recovers. X is visited first.
server.FailingSendPeers.Clear();
server.UnsendableAfter[x.EndPoint] = 0;
Thread.Sleep(30);
Assert.DoesNotThrow(() => server.PollResends());
Assert.That(server.Peers, Is.EqualTo(new[] { y.Server }));
Assert.That(ended, Is.EqualTo(new[] { x.Server }));
Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList();
Assert.That(resent, Has.Count.EqualTo(1));
Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y"));
var goodbye = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Disconnect);
Assert.That(IsUnsendableReason(goodbye.Payload), Is.True);
}
}
[Test]
public void APermanentFailureAcknowledgingAFrameEndsTheSessionAfterTheFrameIsDelivered()
{
var (server, listenerEndPoint, x, _) = TwoPeers();
using (server)
{
var ended = RecordDisconnects(server);
server.UnsendableAfter[x.EndPoint] = 0;
x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);
CultNetRudpSocketServerFrame? frame = null;
Assert.DoesNotThrow(() => frame = server.ReceiveOnce());
Assert.That(frame, Is.Not.Null, "the frame is delivered before the session ends");
Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("from-x"));
Assert.That(ended, Is.EqualTo(new[] { x.Server }));
Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
Assert.That(server.Peers, Has.Count.EqualTo(1));
Assert.That(server.Stats.SendFailures, Is.Zero);
}
}
[Test]
public void APermanentFailureAnsweringARepeatedConnectEndsTheSession()
{
var serverSocket = Bind();
using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
{
RuntimeId = "csharp-send-failure",
Socket = serverSocket,
ConnectionId = ConnectionId,
ResendDelayMs = 5
});
var ended = RecordDisconnects(server);
var peer = Bind();
var connect = CultNetRudpPacketCodec.Encode(
new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0));
peer.SendTo(connect, serverSocket.LocalEndPoint!);
server.ReceiveOnce();
Assert.That(server.Peers, Has.Count.EqualTo(1));
var admitted = server.Peers.Single();
server.UnsendableAfter[peer.LocalEndPoint!] = 0;
peer.SendTo(connect, serverSocket.LocalEndPoint!);
Assert.DoesNotThrow(() => server.ReceiveOnce());
Assert.That(server.Peers, Is.Empty);
Assert.That(ended, Is.EqualTo(new[] { admitted }));
Assert.That(IsUnsendableReason(admitted.DisconnectReason), Is.True);
}
[Test]
public void AConnectWhoseAcceptCanNeverBeSentStartsNoSession()
{
var serverSocket = Bind();
using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
{
RuntimeId = "csharp-send-failure",
Socket = serverSocket,
ConnectionId = ConnectionId,
ResendDelayMs = 5
});
var peer = Bind();
server.UnsendableAfter[peer.LocalEndPoint!] = 0;
peer.SendTo(
CultNetRudpPacketCodec.Encode(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)),
serverSocket.LocalEndPoint!);
Assert.DoesNotThrow(() => server.ReceiveOnce());
Assert.That(server.Peers, Is.Empty);
Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1));
}
[Test]
public void APeerEndedInTheResendLoopIsSentNothingMoreFromIt()
{
var (server, _, x, _) = TwoPeers();
using (server)
{
server.FailingSendPeers.Add(x.EndPoint);
server.SendSchema(x.Server, "one");
server.SendSchema(x.Server, "two");
Assert.That(server.Stats.SendFailures, Is.EqualTo(2));
server.UnsendableAfter[x.EndPoint] = 0;
Thread.Sleep(30);
server.PollResends();
// Both resends are due. The first can never be sent and ends the session; the
// goodbye is the only other datagram attempted at X.
Assert.That(server.Stats.SendFailures, Is.EqualTo(3));
Assert.That(server.Peers, Has.Count.EqualTo(1));
}
}
[Test]
public void AHealthySendCountsItsBytesAndNoFailure()
{
var (server, _, x, _) = TwoPeers();
using (server)
{
var before = server.Stats.BytesSent;
server.SendSchema(x.Server, "hello");
var received = Drain(x.Socket).Sum(p => CultNetRudpPacketCodec.Encode(p).Length);
Assert.That(received, Is.GreaterThan("hello".Length));
Assert.That(server.Stats.BytesSent - before, Is.EqualTo(received));
Assert.That(server.Stats.SendFailures, Is.Zero);
}
}
[Test]
public void OnlyADatagramThatCanNeverBeSentIsAPermanentFailure()
{
foreach (var permanent in new[] { SocketError.MessageSize, SocketError.InvalidArgument, SocketError.AddressFamilyNotSupported })
Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)permanent)), Is.True, permanent.ToString());
foreach (var transient in new[]
{
SocketError.HostUnreachable, SocketError.NetworkUnreachable, SocketError.NoBufferSpaceAvailable,
SocketError.WouldBlock, SocketError.ConnectionReset, SocketError.ConnectionRefused,
SocketError.AccessDenied, SocketError.NetworkDown, SocketError.AddressNotAvailable,
SocketError.Interrupted, SocketError.TimedOut, SocketError.Shutdown
})
Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)transient)), Is.False, transient.ToString());
}
}
}