CultLib/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs

#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using NUnit.Framework;

namespace GameCult.Networking.Tests
{
    /// <summary>
    /// One peer whose datagrams cannot be sent must not stop the listener serving the others. The
    /// failure is injected at the listener's one send function through <c>FailingSendPeers</c>.
    /// </summary>
    public class CultNetRudpSendFailureTests
    {
        private const uint ConnectionId = 0x10203099;

        private sealed class Peer
        {
            public Socket Socket = null!;
            public CultNetRudpSession Session = null!;
            public CultNetRudpSocketServerPeer Server = null!;
            public EndPoint EndPoint => Socket.LocalEndPoint!;
        }

        private static Socket Bind()
        {
            var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
            socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
            return socket;
        }

        private static List<CultNetRudpPacket> Drain(Socket socket)
        {
            var packets = new List<CultNetRudpPacket>();
            var buffer = new byte[65535];
            EndPoint from = new IPEndPoint(IPAddress.Any, 0);
            while (socket.Poll(20_000, SelectMode.SelectRead))
            {
                var received = socket.ReceiveFrom(buffer, ref from);
                packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray()));
            }
            return packets;
        }

        private static Peer Connect(CultNetRudpSocketTransportServer server, EndPoint serverEndPoint)
        {
            var peer = new Peer
            {
                Socket = Bind(),
                Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId })
            };
            peer.Socket.SendTo(CultNetRudpPacketCodec.Encode(peer.Session.CreateConnect(0)), serverEndPoint);
            server.ReceiveOnce();
            peer.Session.Receive(Drain(peer.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
            peer.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(peer.EndPoint));
            return peer;
        }

        private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null, uint? initialSequence = null)
        {
            var serverSocket = Bind();
            var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
            {
                RuntimeId = "csharp-send-failure",
                Socket = serverSocket,
                ConnectionId = ConnectionId,
                ResendDelayMs = 5,
                MaxFragmentBytes = maxFragmentBytes,
                InitialSequence = initialSequence
            });
            var x = Connect(server, serverSocket.LocalEndPoint!);
            var y = Connect(server, serverSocket.LocalEndPoint!);
            return (server, serverSocket.LocalEndPoint!, x, y);
        }

        private static CultNetRudpPacket ReliableFrame(Peer peer, string text) =>
            peer.Session.Send("schema", Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true });

        [Test]
        public void AFailedResendToOnePeerDoesNotStopTheOthersResends()
        {
            var (server, _, x, y) = TwoPeers();
            using (server)
            {
                // Both peers' first datagram is lost, so both have a reliable resend due.
                server.FailingSendPeers.Add(x.EndPoint);
                server.FailingSendPeers.Add(y.EndPoint);
                server.SendSchema(x.Server, "for-x");
                server.SendSchema(y.Server, "for-y");
                Assert.That(server.Stats.SendFailures, Is.EqualTo(2));

                // Y's path recovers; X's stays dead. X is visited first.
                server.FailingSendPeers.Remove(y.EndPoint);
                Thread.Sleep(30);
                Assert.DoesNotThrow(() => server.PollResends());

                var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList();
                Assert.That(resent, Has.Count.EqualTo(1));
                Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y"));
                Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(3));
                Assert.That(server.Peers, Has.Count.EqualTo(2), "a send failure ends no session");
            }
        }

        [Test]
        public void AFailedAckToOnePeerDoesNotStopTheListenerServingTheOthers()
        {
            var (server, listenerEndPoint, x, y) = TwoPeers();
            using (server)
            {
                server.FailingSendPeers.Add(x.EndPoint);
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);
                y.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(y, "from-y")), listenerEndPoint);

                var frames = new List<string>();
                for (var attempt = 0; attempt < 20 && frames.Count < 2; attempt++)
                {
                    Assert.DoesNotThrow(() =>
                    {
                        if (server.ReceiveOnce() is { } frame)
                            frames.Add(Encoding.UTF8.GetString(frame.Frame.Payload));
                    });
                }

                Assert.That(frames, Is.EquivalentTo(new[] { "from-x", "from-y" }));
                Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(1), "X's ack was counted as lost");
                Assert.That(Drain(y.Socket).Select(p => p.PacketType), Does.Contain(CultNetRudpPacketType.Ack));
                Assert.That(server.Peers, Has.Count.EqualTo(2));
            }
        }

        // A permanent send failure is one that can never succeed for the datagram as built. A
        // caller-directed send throws it and queues nothing; inside a poll it ends that peer's
        // session and never the poll.

        [Test]
        public void An_unsendable_goodbye_names_the_failure_by_a_fixed_code_never_its_message()
        {
            var session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 1 });
            Assert.That(
                Encoding.UTF8.GetString(session.EndUnsendable(new SocketException((int)SocketError.MessageSize)).Payload!),
                Is.EqualTo("packet could not be sent: EMSGSIZE"));
            Assert.That(CultNetRudpSession.SendErrorCode(new SocketException((int)SocketError.InvalidArgument)), Is.EqualTo("EINVAL"));
            Assert.That(CultNetRudpSession.SendErrorCode(new SocketException((int)SocketError.AddressFamilyNotSupported)), Is.EqualTo("EAFNOSUPPORT"));
            Assert.That(CultNetRudpSession.SendErrorCode(new SocketException((int)SocketError.ConnectionRefused)), Is.EqualTo("UNKNOWN"));
        }

        private static bool IsUnsendableReason(byte[]? reason)
        {
            var text = Encoding.UTF8.GetString(reason ?? Array.Empty<byte>());
            return text.StartsWith("packet could not be sent: ", StringComparison.Ordinal)
                && text.Length > "packet could not be sent: ".Length;
        }

        private static List<CultNetRudpSocketServerPeer> RecordDisconnects(CultNetRudpSocketTransportServer server)
        {
            var ended = new List<CultNetRudpSocketServerPeer>();
            server.PeerDisconnected += ended.Add;
            return ended;
        }

        [Test]
        public void AnOversizedSendThrowsTheErrorAndQueuesNothing()
        {
            var (server, _, x, y) = TwoPeers();
            using (server)
            {
                // Each peer's own Accept is still awaiting its ack; the refused send adds nothing to it.
                var outstanding = x.Server.Session.OutstandingReliablePacketCount;
                var otherOutstanding = y.Server.Session.OutstandingReliablePacketCount;
                var error = Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[70_000]))!;
                Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString());
                Assert.That(server.Stats.SendFailures, Is.Zero);
                Assert.That(server.Stats.FramesSent, Is.Zero);
                Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session");
                Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing is pending or queued to resend");

                // The refused send consumed no sequence: the next frame is delivered in order instead
                // of waiting behind a gap that never fills.
                server.SendSchema(x.Server, "next");
                var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data);
                var delivered = x.Session.Receive(data, 1).Delivered;
                Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" }));
                Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(otherOutstanding));
            }
        }

        [Test]
        public void ARefusedSendWithdrawsTheFragmentsTheWindowQueuedAsWell()
        {
            var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1);
            using (server)
            {
                var outstanding = x.Server.Session.OutstandingReliablePacketCount;
                server.UnsendableAfter[x.EndPoint] = 0;

                // 1,100 one-byte fragments: the flow window admits the first ones and queues the rest,
                // and the first datagram can never be sent.
                Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[1_100]));

                Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing of the refused send stays queued");
            }
        }

        [Test]
        public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession()
        {
            var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var ended = RecordDisconnects(server);
                server.UnsendableAfter[x.EndPoint] = 1;

                var error = Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[3000]))!;
                Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True);
                // The send ends the session and tells the peer; it leaves the peer for the polling
                // thread to retire, because only that thread changes the peer set.
                Assert.That(x.Server.Connected, Is.False);
                Assert.That(Drain(x.Socket).Last().PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect));
                Assert.That(ended, Is.Empty);

                server.ReceiveOnce();
                Assert.That(server.Peers, Has.Count.EqualTo(1));
                Assert.That(ended, Is.EqualTo(new[] { x.Server }));
                Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
            }
        }

        [Test]
        public void APeerEndedByASendOnAnotherThreadIsRetiredByThePollingThread()
        {
            var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var retiredOn = new List<int>();
                server.PeerDisconnected += _ => retiredOn.Add(Environment.CurrentManagedThreadId);
                server.UnsendableAfter[x.EndPoint] = 1;

                SocketException? thrown = null;
                var sender = new Thread(() =>
                {
                    try { server.SendSchema(x.Server, new byte[3000]); }
                    catch (SocketException error) { thrown = error; }
                });
                sender.Start();
                sender.Join();
                Assert.That(thrown, Is.Not.Null);
                Assert.That(server.Peers, Has.Count.EqualTo(2), "the sending thread leaves the peer set alone");

                server.PollResends();
                Assert.That(retiredOn, Is.EqualTo(new[] { Environment.CurrentManagedThreadId }));
                Assert.That(server.Peers, Has.Count.EqualTo(1));
            }
        }

        // A client that connects again from the same endpoint replaces the peer the listener held. A
        // caller still holding the replaced peer (a handler still running for it, say) reaches nothing
        // of the session that replaced it: no data and no goodbye.

        /// <summary>X connects again as a new session, and the handshake completes.</summary>
        private static (CultNetRudpSocketServerPeer Stale, CultNetRudpSocketServerPeer Replacing) Reconnect(
            CultNetRudpSocketTransportServer server, EndPoint listenerEndPoint, Peer x)
        {
            var stale = x.Server;
            Drain(x.Socket);
            x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId });
            x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint);
            server.ReceiveOnce();
            x.Session.Receive(Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
            x.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint));
            Assert.That(x.Server, Is.Not.SameAs(stale));
            Assert.That(x.Session.Connected, Is.True);
            return (stale, x.Server);
        }

        [Test]
        public void ASendOnAReplacedPeerThrowsAndNoGoodbyeReachesTheSessionThatReplacedIt()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var (stale, replacing) = Reconnect(server, listenerEndPoint, x);
                var ended = RecordDisconnects(server);
                Assert.That(stale.Connected, Is.False, "the replaced peer's session ended with the replacement");

                // Had the stale send gone out, a failure after its first fragment would end the
                // session that sent it and say goodbye to the endpoint.
                server.UnsendableAfter[x.EndPoint] = 1;
                Assert.Throws<InvalidOperationException>(() => server.SendSchema(stale, new byte[3000]));
                server.ReceiveOnce();

                Assert.That(Drain(x.Socket), Is.Empty, "nothing of the stale send reached the client");
                Assert.That(x.Session.Connected, Is.True);
                Assert.That(replacing.Connected, Is.True);
                Assert.That(server.Peers, Has.Member(replacing));
                Assert.That(server.Peers, Has.Count.EqualTo(2));
                Assert.That(ended, Is.Empty);
            }
        }

        // With a fixed initial sequence the stale frame would take the sequence the real one gets, and
        // the real one would be dropped as a duplicate. With a drawn one it would land far ahead of
        // the new session's window, and nothing after it would be delivered.
        [TestCase(1000u)]
        [TestCase(null)]
        public void ASendOnAReplacedPeerDeliversNothingAndTheReplacingPeersFrameArrives(uint? initialSequence)
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(initialSequence: initialSequence);
            using (server)
            {
                var (stale, replacing) = Reconnect(server, listenerEndPoint, x);

                Assert.Throws<InvalidOperationException>(() => server.SendSchema(stale, "STALE"));
                server.SendSchema(replacing, "REAL");

                var delivered = Drain(x.Socket)
                    .SelectMany(packet => x.Session.Receive(packet, 1).Delivered)
                    .Select(frame => Encoding.UTF8.GetString(frame.Payload))
                    .ToList();
                Assert.That(delivered, Is.EqualTo(new[] { "REAL" }));
            }
        }

        // A send that fails after its first fragment ends the generation it began in, and only that
        // one. Here X connects again while the send is under way, and the polling thread admits the
        // new Connect before the fragment that can never be sent. The hook runs that poll inside the
        // send so the order is fixed; in production it runs on the polling thread between the send
        // and the failure's handling. The goodbye belonged to the ended generation, and the client
        // must not hear it: it would believe itself disconnected while the listener holds it
        // connected.
        [Test]
        public void ASendFailureAfterANewConnectReplacedThePeerSaysNoGoodbyeToTheNewSession()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var stale = x.Server;
                Drain(x.Socket);
                x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId });
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint);

                var fragments = 0;
                server.BeforeSend = (remote, packet) =>
                {
                    if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Data || ++fragments < 2)
                        return;
                    server.BeforeSend = null;
                    server.ReceiveOnce();
                    server.UnsendableAfter[x.EndPoint] = 0;
                };
                Assert.Throws<SocketException>(() => server.SendSchema(stale, new byte[3000]));
                server.ReceiveOnce();

                var packets = Drain(x.Socket);
                Assert.That(packets.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect),
                    "the ended generation's goodbye reached the session that replaced it");
                x.Session.Receive(packets.Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0);
                Assert.That(x.Session.Connected, Is.True);
                var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint));
                Assert.That(replacing, Is.Not.SameAs(stale));
                Assert.That(replacing.Connected, Is.True);
            }
        }

        // A caller's send ends X's session over a fragment it could never send. Before the listener
        // retires X, the polling thread answers X's Ping on the ended session, and that Pong can never
        // be sent either. The polling thread read the generation only after the session ended, so the
        // generation alone cannot tell it the session is gone. It says no second goodbye, and the
        // first failure's reason stands. The hook admits the Ping while the first goodbye is on its way,
        // after the session ended and before the peer is queued for retirement.
        [Test]
        public void AFailureAnsweringAPingOnAnAlreadyEndedSessionSaysNoSecondGoodbye()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var ended = RecordDisconnects(server);
                byte[]? firstReason = null;
                server.BeforeSend = (remote, packet) =>
                {
                    if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect)
                        return;
                    server.BeforeSend = null;
                    firstReason = x.Server.UnsendableReason;
                    server.UnsendableAfter[x.EndPoint] = 0;
                    server.ReceiveOnce();
                };
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreatePing(Encoding.UTF8.GetBytes("ping"))), listenerEndPoint);
                server.UnsendableAfter[x.EndPoint] = 1;

                Assert.Throws<SocketException>(() => server.SendSchema(x.Server, new byte[3000]));
                server.ReceiveOnce();

                var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList();
                Assert.That(goodbyes, Has.Count.EqualTo(1));
                Assert.That(firstReason, Is.Not.Null);
                Assert.That(x.Server.UnsendableReason, Is.SameAs(firstReason), "the second failure overwrote the first's reason");
                Assert.That(ended, Is.EqualTo(new[] { x.Server }));
                Assert.That(x.Server.DisconnectReason, Is.SameAs(firstReason));
            }
        }

        // The goodbye for a failed send is sent under the session's gate. The polling thread, admitting
        // a new Connect from X, waits on that gate, so the ended generation's goodbye can never follow
        // the new session's Accept. The hook starts the polling thread just before the goodbye leaves
        // and gives it a second to get ahead if nothing holds it back.
        [Test]
        public void AFailedSendsGoodbyeLeavesBeforeANewConnectFromTheEndpointIsAdmitted()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var stale = x.Server;
                Drain(x.Socket);
                x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId });
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint);

                Thread? poller = null;
                server.BeforeSend = (remote, packet) =>
                {
                    if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect)
                        return;
                    server.BeforeSend = null;
                    poller = new Thread(() => server.ReceiveOnce());
                    poller.Start();
                    poller.Join(1000);
                };
                server.UnsendableAfter[x.EndPoint] = 1;
                Assert.Throws<SocketException>(() => server.SendSchema(stale, new byte[3000]));
                Assert.That(poller, Is.Not.Null);
                poller!.Join();

                var types = Drain(x.Socket).Select(p => p.PacketType).ToList();
                Assert.That(types, Does.Contain(CultNetRudpPacketType.Disconnect));
                Assert.That(types, Does.Contain(CultNetRudpPacketType.Accept));
                Assert.That(types.IndexOf(CultNetRudpPacketType.Disconnect), Is.LessThan(types.IndexOf(CultNetRudpPacketType.Accept)),
                    "the ended generation's goodbye followed the new session's Accept");
            }
        }

        // An application ending a peer's session (a message it could not decode or handle) says no
        // goodbye to a session that already ended: a new Connect replaced it, and the endpoint now
        // belongs to the session that replaced it.
        [Test]
        public void EndingAReplacedPeersSessionSaysNoGoodbyeToTheSessionThatReplacedIt()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers();
            using (server)
            {
                var (stale, replacing) = Reconnect(server, listenerEndPoint, x);
                var ended = RecordDisconnects(server);

                Assert.That(server.EndPeerSession(stale, Encoding.UTF8.GetBytes("application reason")), Is.False);

                Assert.That(Drain(x.Socket).Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect));
                Assert.That(x.Session.Connected, Is.True);
                Assert.That(replacing.Connected, Is.True);
                Assert.That(server.Peers, Has.Member(replacing));
                Assert.That(ended, Is.Empty);
            }
        }

        [Test]
        public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000);
            using (server)
            {
                var ended = RecordDisconnects(server);
                // The polling thread acknowledges X's frame outside the session's gate. Just before
                // that ack goes out, a caller's send to X fails after its first fragment and ends the
                // session; then the ack can never be sent either.
                server.BeforeSend = (remote, packet) =>
                {
                    if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Ack)
                        return;
                    server.BeforeSend = null;
                    server.UnsendableAfter[x.EndPoint] = 1;
                    var caller = new Thread(() =>
                    {
                        try { server.SendSchema(x.Server, new byte[3000]); }
                        catch (SocketException) { }
                    });
                    caller.Start();
                    caller.Join();
                    server.UnsendableAfter[x.EndPoint] = 0;
                };
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);
                server.ReceiveOnce();
                server.PollResends();

                var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList();
                Assert.That(goodbyes, Has.Count.EqualTo(1));
                Assert.That(ended, Is.EqualTo(new[] { x.Server }));
                Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
            }
        }

        [Test]
        public void APermanentFailureInResendsEndsOnlyThatPeersSession()
        {
            var (server, _, x, y) = TwoPeers();
            using (server)
            {
                var ended = RecordDisconnects(server);
                server.FailingSendPeers.Add(x.EndPoint);
                server.FailingSendPeers.Add(y.EndPoint);
                server.SendSchema(x.Server, "for-x");
                server.SendSchema(y.Server, "for-y");

                // X's datagram can now never be sent as built; Y's path recovers. X is visited first.
                server.FailingSendPeers.Clear();
                server.UnsendableAfter[x.EndPoint] = 0;
                Thread.Sleep(30);
                Assert.DoesNotThrow(() => server.PollResends());

                Assert.That(server.Peers, Is.EqualTo(new[] { y.Server }));
                Assert.That(ended, Is.EqualTo(new[] { x.Server }));
                Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
                var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList();
                Assert.That(resent, Has.Count.EqualTo(1));
                Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y"));
                var goodbye = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Disconnect);
                Assert.That(IsUnsendableReason(goodbye.Payload), Is.True);
            }
        }

        [Test]
        public void APermanentFailureAcknowledgingAFrameEndsTheSessionAfterTheFrameIsDelivered()
        {
            var (server, listenerEndPoint, x, _) = TwoPeers();
            using (server)
            {
                var ended = RecordDisconnects(server);
                server.UnsendableAfter[x.EndPoint] = 0;
                x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint);

                CultNetRudpSocketServerFrame? frame = null;
                Assert.DoesNotThrow(() => frame = server.ReceiveOnce());

                Assert.That(frame, Is.Not.Null, "the frame is delivered before the session ends");
                Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("from-x"));
                Assert.That(ended, Is.EqualTo(new[] { x.Server }));
                Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True);
                Assert.That(server.Peers, Has.Count.EqualTo(1));
                Assert.That(server.Stats.SendFailures, Is.Zero);
            }
        }

        [Test]
        public void APermanentFailureAnsweringARepeatedConnectEndsTheSession()
        {
            var serverSocket = Bind();
            using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
            {
                RuntimeId = "csharp-send-failure",
                Socket = serverSocket,
                ConnectionId = ConnectionId,
                ResendDelayMs = 5
            });
            var ended = RecordDisconnects(server);
            var peer = Bind();
            var connect = CultNetRudpPacketCodec.Encode(
                new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0));
            peer.SendTo(connect, serverSocket.LocalEndPoint!);
            server.ReceiveOnce();
            Assert.That(server.Peers, Has.Count.EqualTo(1));
            var admitted = server.Peers.Single();

            server.UnsendableAfter[peer.LocalEndPoint!] = 0;
            peer.SendTo(connect, serverSocket.LocalEndPoint!);
            Assert.DoesNotThrow(() => server.ReceiveOnce());

            Assert.That(server.Peers, Is.Empty);
            Assert.That(ended, Is.EqualTo(new[] { admitted }));
            Assert.That(IsUnsendableReason(admitted.DisconnectReason), Is.True);
        }

        [Test]
        public void AConnectWhoseAcceptCanNeverBeSentStartsNoSession()
        {
            var serverSocket = Bind();
            using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions
            {
                RuntimeId = "csharp-send-failure",
                Socket = serverSocket,
                ConnectionId = ConnectionId,
                ResendDelayMs = 5
            });
            var peer = Bind();
            server.UnsendableAfter[peer.LocalEndPoint!] = 0;
            peer.SendTo(
                CultNetRudpPacketCodec.Encode(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)),
                serverSocket.LocalEndPoint!);

            Assert.DoesNotThrow(() => server.ReceiveOnce());

            Assert.That(server.Peers, Is.Empty);
            Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1));
        }

        [Test]
        public void APeerEndedInTheResendLoopIsSentNothingMoreFromIt()
        {
            var (server, _, x, _) = TwoPeers();
            using (server)
            {
                server.FailingSendPeers.Add(x.EndPoint);
                server.SendSchema(x.Server, "one");
                server.SendSchema(x.Server, "two");
                Assert.That(server.Stats.SendFailures, Is.EqualTo(2));

                server.UnsendableAfter[x.EndPoint] = 0;
                Thread.Sleep(30);
                server.PollResends();

                // Both resends are due. The first can never be sent and ends the session; the
                // goodbye is the only other datagram attempted at X.
                Assert.That(server.Stats.SendFailures, Is.EqualTo(3));
                Assert.That(server.Peers, Has.Count.EqualTo(1));
            }
        }

        [Test]
        public void AHealthySendCountsItsBytesAndNoFailure()
        {
            var (server, _, x, _) = TwoPeers();
            using (server)
            {
                var before = server.Stats.BytesSent;
                server.SendSchema(x.Server, "hello");
                var received = Drain(x.Socket).Sum(p => CultNetRudpPacketCodec.Encode(p).Length);
                Assert.That(received, Is.GreaterThan("hello".Length));
                Assert.That(server.Stats.BytesSent - before, Is.EqualTo(received));
                Assert.That(server.Stats.SendFailures, Is.Zero);
            }
        }

        [Test]
        public void OnlyADatagramThatCanNeverBeSentIsAPermanentFailure()
        {
            foreach (var permanent in new[] { SocketError.MessageSize, SocketError.InvalidArgument, SocketError.AddressFamilyNotSupported })
                Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)permanent)), Is.True, permanent.ToString());
            foreach (var transient in new[]
            {
                SocketError.HostUnreachable, SocketError.NetworkUnreachable, SocketError.NoBufferSpaceAvailable,
                SocketError.WouldBlock, SocketError.ConnectionReset, SocketError.ConnectionRefused,
                SocketError.AccessDenied, SocketError.NetworkDown, SocketError.AddressNotAvailable,
                SocketError.Interrupted, SocketError.TimedOut, SocketError.Shutdown
            })
                Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)transient)), Is.False, transient.ToString());
        }
    }
}