diff --git a/.github/workflows/cultnet-interop.yml b/.github/workflows/cultnet-interop.yml index 5f8936bf..9072e0cb 100644 --- a/.github/workflows/cultnet-interop.yml +++ b/.github/workflows/cultnet-interop.yml @@ -12,6 +12,13 @@ jobs: name: C# / TypeScript / Rust / Python / Kotlin runs-on: windows-latest timeout-minutes: 60 + # Windows runners default to pwsh, which does not stop when a native command fails: a + # multi-command step reported only its last command, and a failing cultnet-py suite + # went green that way. Git Bash runs as `bash -e -o pipefail`, so every command counts. + # Steps that name `shell: pwsh` check $LASTEXITCODE themselves. + defaults: + run: + shell: bash env: DOTNET: dotnet @@ -69,7 +76,11 @@ jobs: shell: pwsh run: | $ErrorActionPreference = "Stop" - $stray = git ls-files '*.schema.json' | Where-Object { $_ -notmatch '^contracts/cultnet/' } + # packages/cultnet-rs/contracts/cultnet/ holds R-AB's vendored byte copies (the crate must build + # when copied alone). Their drift guard is packages/cultnet-rs/tests/schema_discovery_contracts.rs:74-99. + $tracked = git ls-files '*.schema.json' + if ($LASTEXITCODE -ne 0) { throw "git ls-files failed" } + $stray = $tracked | Where-Object { $_ -notmatch '^(packages/cultnet-rs/)?contracts/cultnet/' } # -I skips binary files: unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll # still embeds the retired namespace until it is rebuilt (REC-032 D4, deferred). $oldNamespace = git grep -I -lE 'GameCult/cultnet-ts/(integration/)?contracts' -- ':!.voidbot' diff --git a/docs/cultnet-error-contract-cut.md b/docs/cultnet-error-contract-cut.md new file mode 100644 index 00000000..5673d1d4 --- /dev/null +++ b/docs/cultnet-error-contract-cut.md @@ -0,0 +1,656 @@ +# CultNet error contract: cut map + +Status: cut map, not started. Imagination pass 1, 2026-09-30, against CultLib `main` at `8fc74c70`. +This document owns the means. There is no separate target doc: section 1 holds the ends until the +operator rules on section 3. After that, Self moves the rulings and invariants into +`contracts/cultnet/cultnet-distributed-database.md` ("Error Codes") and the schema's own description. + +Open: operator questions Q1-Q8 (section 3). No cut is dispatched until Q1, Q2, Q3 and Q8 are ruled. Q4-Q7 +change only single emit sites and can be ruled during Cut 1. + +Depends on: `hands/cultmesh-put-serve-bound` merging first. It introduces `document_unservable` in Python +(`fd394d54`), a codeless unservable refusal in cultmesh-ts (`0eb7434a`) and `CultMeshRudpRejectionReason` in +cultmesh-rs (`c9cdad12`, `85288a5a`). The cuts below convert those sites. Their anchors are by commit, not by +line, because the branch is not on main. + +Found by the put-serve Hands (batch 3, `38000efa`, 2026-09-30); they belong to this map's cuts: +- **Python codes outside the closed code list.** Python refuses with `document_unservable`, + `malformed_document_put` and `snapshot_*_exceeded`. None of these are in the closed `cultnet.error.v0` code + list, and Rust and TS peers refuse to decode an unknown code. +- **Two error-map shapes.** TS sends a two-key error map that its own peers cannot parse. Rust sends the five-key + contract shape. The two agree only at the message level. +- **The session after a refusal.** TS and Python keep the session open after a refusal; Rust sends a goodbye and + ends it. +- **New Rust refusal frame.** The put-serve branch adds a Rust refusal frame, `cultnet.error.v0` carrying the + reason text with no code and no details, sent before the goodbye and acknowledging nothing. Cut 1 gives it a + code. + +Map branch: `main`. + +--- + +## 0. What the Body does today (probed) + +Every claim in this table was established by running code. The C#, Rust and TypeScript probes ran on +Yggdrasil through `ygg-verify.sh` at `8fc74c70`. The Python bytes came from `msgpack.packb` on the exact +dicts `cultmesh_py/server.py:573-589` and `cultmesh-ts/src/index.ts:5921-5926` build. The six probe inputs: + +| Input | Bytes from | +|---|---| +| `cs_cursor_stale` | Rust `tests/error_message.rs` captured C# hex (`cursor_stale`, details `{asOf:1,current:2}`) | +| `cs_routing` | C# `new CultNetErrorMessage { Error, RoutingHint = { Reason="not_primary", Shard=... } }` serialized with `CultNetSchemaMessageSerialization` | +| `cs_codeless` | C# `new CultNetErrorMessage { Error = "boom" }` | +| `cs_unknown_code` | C# with `Code = "document_unservable"`, empty `CultNetErrorDetails` | +| `py_snapshot_byte_limit` | the Python `_error_response` shape: `messageId`, `code: "snapshot_byte_limit_exceeded"`, `details {responseBytes, maxSnapshotBytes}` | +| `ts_two_key` | the cultmesh-ts shape `{schemaVersion, error}` | + +Decoded by: + +| Input | C# `CultNetSchemaMessageSerialization.Deserialize` | Rust `decode_cultnet_message_from_slice` | TS `parseCultNetMessage` | +|---|---|---|---| +| `cs_cursor_stale` | ok | ok, typed | ok | +| `cs_routing` | ok | ok, **routingHint dropped** (`code: None, details: None`) | ok (untyped passthrough) | +| `cs_codeless` | ok | ok | ok | +| `cs_unknown_code` | ok (free string) | **Err: "not a recognized cultnet.error.v0 code"** | **throws: /code must be one of the allowed values** | +| `py_snapshot_byte_limit` | ok, but **details silently become an all-null object** (`responseBytes`, `maxSnapshotBytes` lost), `messageId` dropped | **Err: not a recognized code** | **throws** (missing routingHint, extra messageId, code enum, details keys) | +| `ts_two_key` | ok | ok | **throws: missing routingHint, code, details** — TypeScript cannot parse the errors TypeScript sends | + +Captured C# bytes for the two shapes Rust's vector file does not hold (for Hands' reuse): + +- `cs_codeless`: `85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72A4626F6F6DAB726F7574696E6748696E74C0A4636F6465C0A764657461696C73C0` +- `cs_routing`: see the probe; the shard descriptor writes ten keys, nil included. + +Other findings, from reading source at `8fc74c70`: + +1. **The interop CI lane has been red for 40 consecutive `main` runs.** It fails at "Verify CultNet schema + contracts stayed consolidated" (`.github/workflows/cultnet-interop.yml:67-90`). That step forbids every + `*.schema.json` outside `contracts/cultnet/`, and R-AB deliberately vendors 22 copies into + `packages/cultnet-rs/contracts/cultnet/`. Those copies are held byte-equal by + `packages/cultnet-rs/tests/schema_discovery_contracts.rs:74-99`. Nothing after that step (every + cross-runtime interop test) has run on main in that time. Run `36714768445` is the latest. +2. **The schema contradicts itself.** `contracts/cultnet/cultnet.error.schema.json:29` says the code is + "Additive: a v0 peer that does not know this field ignores it" and, in the same sentence, "a closed + enum: Rust and TypeScript refuse". The closed-enum text came from Soul pass 4d (`3f942a57`). It + described what the code did; no operator ruled it. `docs/` holds no ruling on unknown codes. +3. **Emitters that break the schema:** + - cultmesh-ts sends `{schemaVersion, error}` only, at `packages/cultmesh-ts/src/index.ts:5875-5878`, + `5885-5888`, `5913-5916` and `5922-5925`, and on the branch at `0eb7434a`. + - cultmesh-py `_error_response` (`packages/cultmesh-py/src/cultmesh_py/server.py:573-589`) adds a + top-level `messageId`, omits `routingHint`, omits `code` and `details` rather than sending nil, and + sends eight out-of-enum codes: `unsupported_schema_version` (`:161`), `malformed_document_put` + (`:358`), `unregistered_document_put` (`:365`), `malformed_document_delete` (`:386`), + `unregistered_document_delete` (`:393`), `snapshot_document_limit_exceeded` (`:558`), + `snapshot_byte_limit_exceeded` (`:567`) and `simulation_observations_disabled` (`:596`). The branch + adds a ninth, `document_unservable` (`fd394d54`). + - `packages/cultnet-py/src/cultnet_py/interop_peer.py:409-417` also sends `unsupported_schema_version`. + R-2 (`docs/cultnet-selection-cut.md:2046-2053`) required that code, but it never entered the schema + enum. +4. **A second schema authority.** `packages/cultnet-py/src/cultnet_py/schema_catalog.py:17`, `:272` and + `:357-362` publish a hand-built `cultnet.error.v0` descriptor under the canonical `$id`. It has a + different shape: `messageId`, `code` as any string, `details` open, and only `schemaVersion` and `error` + required. The same file does this for every wire message (about 350 lines). This is recorded as + follow-up F-1, not cut here. +5. **Silent refusals.** cultmesh-rs's RUDP document server returns every rejection to the local caller only + (`packages/cultmesh-rs/src/rudp_document_server.rs:470-590`, `CultMeshRudpApplicationRejection`). The + peer gets nothing and times out. On the branch this includes the new `DocumentUnservable`. +6. **Receivers drop the code.** + - Every C# receiver turns an error into `new IOException(error.Error)` or + `new InvalidOperationException(error.Error)`, so a caller cannot branch on the code. The sites: + `src/GameCult.Mesh/CultMeshBodySessions.cs:310`, `CultMeshContentSessions.cs:178/256-260`, + `CultMeshPeerExchange.cs:430/466`, `CultMeshSnapshots.cs:91/107/222-226/743` and + `CultMeshVerseDiscovery.cs:273`, plus `src/GameCult.Networking/CultNetShardReplication.cs:440/574`. + - Rust reads `Error { error, .. }` at `packages/cultmesh-rs/src/lib.rs:823`. + - TS types `CultNetErrorMessage` as `{schemaVersion, error}` only (`packages/cultnet-ts/src/contracts.ts:167-170`). + - Kotlin has no error type. A request that gets an error back fails a + `require(response.schemaVersion == ...)` (`packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt:1278`, + `:1334`), and the peer's text is lost. +7. **No correlation.** Only Python echoes `messageId`. + - `cultmesh-browser` rejects an error only when exactly one operation is pending + (`packages/cultmesh-browser/src/index.ts:724-731`); otherwise the error is dropped and the operation + times out. + - C# `CultMeshContentSessions.OnError` (`:256-260`) fails every pending request on any error. +8. **The reference is inconsistent with itself.** `src/GameCult.Networking/CultNetDatabaseServer.cs:560` + refuses a variant change with a codeless error. The subscription server refuses the same condition with + `ForVariantUnsupported` (`CultNetDatabaseSubscriptionServer.cs:211/333/366`). +9. **routingHint is never consumed.** No receiver in any runtime reads it. C# builds it at + `CultNetDatabaseServer.cs:638-650`, and Rust's decode discards it. +10. **Consumer audit** (read-only, all of `F:\Projects` except CultLib; no git in Aetheria): + - Outside CultLib, nothing sets `code`, `details` or `routingHint`, and nothing branches on a code. + - Emitters: + - Huginn: `crates/huginn-daemon/src/serve.rs:288/307/312`, Rust struct literals + `Error { error, code: None, details: None }`, pinned at cultnet-rs `8fc74c70`. + - AetheriaEve: `Aetheria.State.Daemon/Program.cs:1737/1778`, C# object initializers. + - Decoders read `.error` only: Gjallar `Program.cs:574`, Mimir + `MimirOdinMoveProofEvidenceRingProvider.cs:80`, Hermodr `hermodr-daemon.cjs:1380` (hand-rolled), + AetheriaEve's smoke test (text match against its own emit text), and Huginn's tests (text match). + - Stale vendored TS: + - `StreamPixels/vendor/CultLib` carries the current closed schema, so any change here breaks its + TS error decode until it is re-vendored. + - `Heimdall/vendor/CultLib` carries a pre-R-X schema with `additionalProperties:false` and no + `routingHint`, so it already refuses every error C# sends today. + - Odin, Idunn, Muninn, Ghostlight and voidbot: zero hits. + +## 1. Ends + +- **One schema.** `contracts/cultnet/cultnet.error.schema.json` is the only authority on the shape and on the + code registry. C# embeds it (`GameCult.Networking.csproj:19`), TS mirrors it at build time, and Rust + vendors a copy held byte-equal by test. Nothing else publishes a `cultnet.error.v0` shape (F-1 covers + Python). +- **Every runtime emits only schema-valid errors.** This is proven per emit path against the schema, and + byte-equal to the C# reference for every registry code. +- **Every runtime decodes every error it can receive into a typed value.** A known code becomes its typed + variant with its typed details. An unknown code, if Q1 says so, becomes a typed "other" that keeps the + code string and the raw details. No well-formed error is ever refused, and no field is silently dropped + (routingHint, messageId). +- **Code and details have one owner.** The details a message carries are decided by its code. The types in + every runtime make "cursor_stale with a field" unrepresentable. No free-standing code string plus a + grab-bag details object. +- **A refusal is an answer, not a silence** (R-2 precedent, and Q4). A server that refuses a peer's request, + and can still send, sends a coded error that correlates to the request. +- **The public surface is complete in every runtime**, per the consumer-expectation doctrine: + - a typed error value; + - a constructor per registry code; + - a typed exception or error that request APIs raise when the peer refuses, carrying the typed value. + +## 2. Identity, lifecycle and authority + +| Kind | What names it | Lifecycle | Who decides | +|---|---|---|---| +| Error message | `schemaVersion: cultnet.error.v0`; correlated to its request by `messageId` (Q3) | Sent once, in reply to one request or subscription. Never persisted. | The server that refused. | +| Error code | A snake_case string in the schema's registry (`$defs/knownCode`) | Added by one coordinated change: schema, C# constructor, a C#-written vector, and a typed variant in every runtime. Never renamed and never reused. A retired code stays in the registry marked retired, so an old reader still decodes it. | The operator rules meaning; Self keeps the registry. | +| Details for a code | The code | Frozen with the code. Adding a key to an existing code's details is a new code (or a v1), never an in-place edit (see Q2). | Same as the code. | +| Unknown code, at a reader | The raw string | Preserved as typed `Other`. The reader acts on the `error` text and the absence of a known code. | The reader (Q1). | +| Vector file | `contracts/cultnet/interop/error-vectors.cs-written.json` | Regenerated only by the C# writer test under `CULTNET_WRITE_VECTORS=1` and committed. Every other runtime judges it read-only. | C#, the reference. | +| routingHint | Shard authority refusal | Sent with a null code (unchanged). Typed in every decoder. | CultNetDatabaseServer's authority path. | + +## 3. Operator questions + +**Rulings (operator, 2026-09-30):** "Q1: B / Q2: B / Q3: A / Q8: A". +- **Q1:** closed for emitters, open for readers. An unknown code decodes to a typed `Other`. +- **Q2:** per-code details. +- **Q3:** a nullable top-level `messageId`. +- **Q8:** v0 is corrected in place, with the semver bumps the policy requires. +- Q4-Q7 are still open. +- Cut 1 waits for `hands/cultmesh-put-serve-bound` to merge. + +### Q1. What does a reader do with an error code it does not know? + +Today Rust and TypeScript refuse the whole message. The probes above show the result: the peer's error text +is lost and the caller sees "not a recognized code". In cultmesh-rs's snapshot client the `?` at `lib.rs:812-815` +even turns a peer refusal into a local decode failure. So under a closed enum every new code breaks every +reader that has not upgraded. By `docs/semver-policy.md:86-91` that makes each new code a breaking wire +change, and nine codes are already in flight. + +- **A. Closed enum (today).** Readers refuse unknown codes. Every code addition is a coordinated, + semver-major wire change across all runtimes and consumers. +- **B. Closed for emitters, open for readers.** The schema keeps the registry, and every runtime's emit + constructors can produce only registry codes (tests prove each emit path valid against the schema). + Readers decode an unrecognized code (matching `^[a-z][a-z0-9_]*$`) into a typed `Other { code, details }`. + A new code is an additive, minor change: old readers still surface the text and know "an error I cannot + classify". This is how gRPC status codes and protobuf open enums behave. +- **C. Fully open.** No registry; any string, emit and decode. + +**Recommended: B.** A refusal must never fail to arrive because it is more specific than the reader +expected. B keeps the registry as the single meaning authority. C loses the meaning authority, and A makes +the error channel the most brittle message in CultNet. + +Depends on this: schema `code` shape, every decoder, the doc text at `cultnet-distributed-database.md:107-115`. + +### Q2. How are details shaped on the wire? + +Today C# writes all four detail keys (`field`, `value`, `asOf`, `current`), nil included, because +`CultNetErrorDetails` (`CultNetSchemaMessages.cs:592-603`) is a MessagePack map-mode object. The codes in +flight need new keys: `responseBytes`, `maxSnapshotBytes`, `documentCount`, `maxSnapshotDocuments` and +`schemaVersion`. + +- **A. One shared details record; grow it.** Every detail key of every code sits on one record, written as + nil when unused. Each new key changes the bytes of every existing error that has details, so all of the + C# vectors must be recaptured on every addition. It also breaks every strict reader: TS Ajv, and + StreamPixels' vendored schema, have `additionalProperties:false`. And the type cannot tie a key to its + code. +- **B. Per-code details.** `details` carries exactly the keys its code defines. A defined-but-optional key + is present as nil; another code's keys are absent. Keys are named for their meaning, which retires the + `field: "schemaId"` / `field: "subscriptionId"` indirection that exists only because the record is + shared. Adding a code never changes an existing code's bytes. In every runtime, code and details are + fused into one tagged type: + - Rust: an enum with data. + - C#: sealed subclasses, with a custom formatter for `CultNetErrorMessage`. There is precedent in + `src/GameCult.Mesh/CultMeshRouteRecordFormatter.cs`. + - TS: a discriminated union. + - Kotlin: a sealed class. + - Python: constructors per code over a frozen dataclass. + + The churn is one time. The five existing C# cases with details change bytes, and all eight Rust hex + cases move into the shared vector file anyway (Cut 1). +- **C. Opaque string map.** Untyped; no runtime can offer a typed accessor. + +**Recommended: B.** A is the churn the brief warned of, and it recurs on every future code. B pays it once, +at the moment every vector is being recaptured regardless. + +Depends on this: the schema's `details`, the C# model and formatter, `encode_error_details` +(`packages/cultnet-rs/src/contracts.rs:1339-1366`), and every decoder. + +### Q3. Does an error carry the `messageId` of the request it refuses? + +Python already sends `messageId` (`server.py:583`); the schema forbids it. Without it: + +- `cultmesh-browser` drops an error whenever two operations are pending (`index.ts:724-731`); +- C# content sessions fail every pending request on any error (`CultMeshContentSessions.cs:256-260`). + +Adding a top-level key changes the bytes of every C# error. That happens once, in the same recapture as Q2. + +- **A.** Add a nullable top-level `messageId`. A reply to a request echoes the request's id, and an error + with no request (a subscription push, an unsolicited refusal) sends nil. Receivers correlate by it and + fall back to their current behavior when it is nil. +- **B.** No correlation. Python deletes the field. + +**Recommended: A.** Correlation is a capability any multiplexing client needs, and one runtime already +depends on it. + +### Q4. Do servers send `document_unservable` (and every other refusal they can) to the peer? + +Self ruled that "unservable" must be distinguishable as a typed code in every runtime. On the branch: +Python sends it coded, TypeScript sends it codeless, and Rust tells only its local caller, so the Rust +peer times out. The same Rust silence covers every other document-server rejection: a sink refusal, a +failed snapshot source, too many snapshot documents, and a snapshot response that is too large. + +- **A.** Every refusal of a peer's request that the server can still send is answered with a coded error + carrying the request's `messageId`, and is also reported locally (Rust's `ApplicationRejected` stays as + the local report). The refusals that cannot be answered are the ones where the answer itself cannot be + queued: Rust `PayloadBudgetFull` and `ResponseQueueFailed`. Those stay local-only, and the doc says so. +- **B.** Local-only in Rust; peer replies only where a runtime already sends them. + +**Recommended: A.** R-2 already ruled that a silence is worse than a missing feature +(`docs/cultnet-selection-cut.md:2049-2053`), and the operator's ruling that CultMesh "refuses" puts only +means something to the party that sent the put. + +### Q5. Does Python's `unregistered_document_put` / `unregistered_document_delete` become `unowned_schema`? + +Python sends these when the node has no registered document type for the record's schemaId +(`cultmesh_py/node.py:653-666`). C# `unowned_schema` means "no shard owns this schema and key" +(`CultNetSchemaMessages.cs:537-544`). To a peer both mean "this server will not take that schema here; route +elsewhere". The put and delete variants duplicate what the peer already knows (it sent the message). + +- **A.** Both collapse into `unowned_schema { schemaId }`. The registry text broadens to "this server + stores no document for that schema (and key)". +- **B.** Join as a separate `schema_unregistered { schemaId }`. +- **C.** Join both codes as they are. + +**Recommended: A.** It is one meaning at the peer, and there is one action to take. + +### Q6. Does Python's `malformed_document_put` / `malformed_document_delete` become one `message_invalid { field }`? + +These are sent when the request lacks a required field (`server.py:355-359`, `383-387`). C# has the same +condition, codeless, at `CultNetDatabaseServer.cs:373/415` and `CultNetDatabaseSubscriptionServer.cs:148` +("requires a subscriptionId or messageId"). + +- **A.** One `message_invalid { field }`, naming the missing or ill-typed field. It is used by Python's four + sites and C#'s three. +- **B.** Join both per-message codes and leave C# codeless. + +**Recommended: A.** The per-message suffix repeats the request's own schemaVersion, and C# gains a code for +free. + +### Q7. Does `simulation_observations_disabled` become `unsupported_schema_version`? + +Python sends it when the observation hub is off (`server.py:593-598`), with +`details { schemaVersion: "cultnet.simulation_observation.v0" }`, which is exactly the +`unsupported_schema_version` shape. cultmesh-ts's "no cache for snapshot requests" and "no operation +handler" refusals (`index.ts:5875`, `5913`) mean the same thing: this server does not serve that message. + +- **A.** Collapse. `unsupported_schema_version` means "this server does not serve messages of this + schemaVersion: unknown, a newer version, or not enabled here". The three TS sites and Huginn's "answers + only X, Y, Z" adopt it. +- **B.** Keep a separate `feature_disabled`-style code. + +**Recommended: A.** A peer cannot act differently on "not enabled" and "not known". Python's +`verify.py:335` check follows the rename. + +### Q8. Is this a correction of `cultnet.error.v0` in place, or a new `cultnet.error.v1`? + +Under Q2-B and Q3-A, strict v0 readers (TS Ajv and vendored copies) refuse the new bytes. Tolerant readers +(C#, Rust, Python) keep working, because the probes show they ignore unknown keys and treat a missing key +as null. + +- **A. In place.** The v0 contract has never been decodable across all five runtimes (section 0). No + consumer outside CultLib reads code or details (finding 10). Every publishing package takes the semver + bump its policy requires: + - `org.gamecult.cultlib` major, because the public `CultNetErrorDetails` properties change; + - `cultnet-py` and `cultmesh-py` major, because emitted codes change; + - Rust pins move deliberately. + + StreamPixels and Heimdall re-vendor. +- **B. v1.** Add `cultnet.error.v1`, keep v0 emit for peers that did not advertise v1 in hello, and run + two error encoders in every runtime until v0 retires. + +**Recommended: A.** B builds a version-negotiation path for a message nobody outside CultLib parses beyond +`.error`. The one thing every old reader does need, the `error` text, survives A in every tolerant runtime. + +## 4. The registry (as recommended; rewrite on each ruling) + +| Code | Meaning | Details (per Q2-B; `?` = present-as-nil when absent) | Emitted by after this map | +|---|---|---|---| +| `selection_invalid` | A selection field failed the door. | `{ field, value? }` | C#, Rust (selection) | +| `cursor_stale` | The cursor predates the server's retained history. | `{ asOf, current }` | C#, Rust | +| `cursor_invalid` | The cursor could not be parsed. | none (details nil) | C#, Rust | +| `reference_outside_target` | A hop reference left the declared target. | none | C#, Rust | +| `variant_unsupported` | The selection selects a document variant CultNet does not carry. | `{ subscriptionId? }` | C# (incl. `DatabaseServer.cs:560`) | +| `unowned_schema` | This server stores no document for that schema (and key). | `{ schemaId }` | C#, Python (Q5) | +| `unsupported_schema_version` | This server does not serve messages of this schemaVersion. | `{ schemaVersion }` | Python, TS, Huginn (Q7) | +| `message_invalid` | The request lacks a required field or has an ill-typed one. | `{ field }` | C#, Python (Q6) | +| `snapshot_document_limit_exceeded` | The snapshot answer would exceed the document bound. | `{ documentCount, maxSnapshotDocuments }` | Python, Rust (Q4) | +| `snapshot_byte_limit_exceeded` | The snapshot answer would exceed the byte bound. | `{ responseBytes, maxSnapshotBytes }` | Python, Rust (Q4) | +| `document_unservable` | A put was refused because no snapshot could ever return the stored document under the byte bound. | `{ responseBytes, maxSnapshotBytes }` | Python, TS, Rust (Q4) | + +A null code is legal and means "unclassified". Examples: an exception message from a failed snapshot +source, an authority refusal carrying `routingHint`, and "not authorized" +(`CultNetDatabaseSubscriptionServer.cs:165`). This map does not invent codes for those. + +`snapshot_byte_limit_exceeded` and `document_unservable` have identical details. That is deliberate, and it +is the fixture trap named in the verification rules. + +--- + +## 5. Cuts + +All cuts are made on one campaign branch, `hands/error-contract`, from `main` after +`hands/cultmesh-put-serve-bound` merges. The branch merges to `main` once, after Cut 5's Soul pass. The +reason: Cut 1 changes C# bytes, and Rust's static hex cases would stay green against the stale bytes +(finding: `tests/error_message.rs` pins constants, not the reference). A partial merge would ship a +wire split while every suite passed. Each cut gets its own worktree, created by Self. Heavy verification +runs through `ygg-verify.sh`, never on Starfire. + +### Cut 0. Unblock the interop lane + +- **Repo/branch:** CultLib, `hands/interop-guard` from `main`, merged alone ahead of the campaign. It + does not depend on the rulings. +- **Deletes first:** nothing. +- **Per-file changes:** `.github/workflows/cultnet-interop.yml:72`. The `$stray` filter also accepts + `^packages/cultnet-rs/contracts/cultnet/`, with a comment naming R-AB and + `packages/cultnet-rs/tests/schema_discovery_contracts.rs:74-99` as the equality guard. The namespace + check is unchanged. +- **Authority map:** no ownership change. The canonical owner stays `contracts/cultnet/`, and the Rust + mirror is derived from it, with drift judged by the Rust test. +- **Verification:** + - Push the branch and run the workflow. Report every step that goes red after the guard. None of those + steps has run on main for 40 runs, so expect more red, and treat each failure as its own finding; + do not fix it in this cut. + - Negative: edit one byte of a vendored copy locally and check that the Rust equality test fails. + Reach it on Yggdrasil with `cd packages/cultnet-rs && cargo test --test schema_discovery_contracts`. +- **Ledger:** +3 lines. + +### Cut 1. The contract and the C# reference + +- **Repo/branch:** CultLib `hands/error-contract`, worktree named by Self. +- **First:** capture the current C# bytes for all eight factory cases, plus `cs_codeless` and `cs_routing`, + into the Hands report. This is the before-image a Soul pass compares against. +- **Deletes first:** + - `CultNetErrorDetails` (`src/GameCult.Networking/CultNetSchemaMessages.cs:588-603`, 16 lines) as a + shared record. + - The `field`/`value`/`asOf`/`current` property block and the `[MessagePackObject]`/`[Key]` attributes + on `CultNetErrorMessage` (`:497-521`). Its serialization moves to one formatter. + - `SoulP4dProbeTests.Wire_UnownedSchemaBytes` (`tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs:191-199`). + It is an `[Explicit]` hex printer, superseded by the vector writer. + - The closed-enum paragraph at `contracts/cultnet/cultnet-distributed-database.md:107-115` and + the `code`/`details` descriptions in the schema (`:28-42`). +- **Adds:** + - **Schema** (`contracts/cultnet/cultnet.error.schema.json`, rewritten; the Rust mirror is re-copied + byte-equal): + - Top level: `schemaVersion`, `error` and `routingHint` as today, plus `messageId: string|null` (Q3) + and `code`. All five are required, present-as-nil (R-X is unchanged at the top level). + - `$defs/knownCode` is the registry enum from section 4. + - `code` is `null`, or a string matching `^[a-z][a-z0-9_]*$` (Q1-B). + - `details` is decided by a top-level `oneOf`, one branch per known code, each fixing `code` by + `const` and `details` by exact `properties`/`required`/`additionalProperties:false`. Two more + branches: a null code with null details, and an unknown code (`not: {$ref knownCode}`) with details + `object|null`, open. + - Use `oneOf`/`not`/`const` only. `tests/GameCult.Networking.Tests/MiniJsonSchemaValidator.cs:30-35` + implements those and **not** `if`/`then`/`allOf`, and it refuses unimplemented keywords. TS + compiles with Ajv2020 (`packages/cultnet-ts/src/contracts.ts:1,674`), which handles both. + - **C# model** (`GameCult.Networking`): + - `CultNetErrorMessage { SchemaVersion, Error, MessageId?, RoutingHint?, Refusal? }`, where + `Code => Refusal?.Code` is derived and not settable. + - `abstract CultNetRefusal { string Code }`, with one sealed subclass per registry code carrying its + typed details (`CultNetCursorStaleRefusal(ulong AsOf, ulong Current)` and so on), plus + `CultNetUnknownRefusal(string Code, IReadOnlyDictionary? Details)` for Q1-B. + - `CultNetErrorCodes`: string constants, one per registry code. + - `CultNetErrorMessageFormatter : IMessagePackFormatter`. It writes the five + top-level keys in order (schemaVersion, error, messageId, routingHint, code, details), and details + as exactly the refusal's keys. It reads by key name, tolerates unknown keys at both levels, and + maps an unknown code to `CultNetUnknownRefusal`. Register it through `[MessagePackFormatter]` on + the type, the way `CultMeshRouteRecordFormatter` is wired. + - Factories: keep `ForSelectionInvalid`, `ForCursor`, `ForReferenceOutsideTarget`, + `ForVariantUnsupported` and `ForUnownedSchema` as thin constructors over the refusal types, and add + one per new code. Each takes an optional `messageId`. + - `CultNetPeerErrorException : IOException`, carrying the `CultNetErrorMessage`. + - **Vector writer:** `ErrorVectorTests.WriteErrorVectors`, gated on `CULTNET_WRITE_VECTORS=1` like + `SelectionParityVectorTests.cs:460-552`. It writes `contracts/cultnet/interop/error-vectors.cs-written.json`: + one entry per registry code (both optional-key states where a key is optional), `cs_codeless`, + `cs_routing`, one unknown code with open details, and one error with `messageId` set. Each entry + holds `{ name, messagePackBase64, expected: { error, messageId, code, details, routingHint } }`, + where `expected` is a JSON projection every runtime compares its typed decode against. +- **Per-file changes** (`8fc74c70`): + - `CultNetDatabaseServer.cs:560`: `ForVariantUnsupported(refusal.Message, subscriptionId)`. + - `CultNetDatabaseServer.cs:373`, `:415` and `CultNetDatabaseSubscriptionServer.cs:148` become + `message_invalid { field: "subscriptionId" }` (Q6-A). + - Every `SendCultNet(CultNetErrorMessage...)` that answers a request passes the request's `MessageId`. + - Each receiver listed in finding 6 throws `new CultNetPeerErrorException(error)` instead of + wrapping `error.Error`. `CultMeshContentSessions.OnError` (`:256-260`) fails only the pending entry + whose id matches `error.MessageId`, and falls back to all entries only when the id is nil. + - `CultNetSchemaContractTests.cs:165-214`: the per-factory tests are replaced by one test that validates + every vector's decoded JSON against the schema with `MiniJsonSchemaValidator`, and one test that + serializes each registry factory and asserts byte equality with its committed vector. + - Rewrite `contracts/cultnet/cultnet-distributed-database.md` "Error Codes" to the registry, and state + the Q1 and Q2 rules. +- **Authority map:** + - Owner: the schema's registry owns code meaning and detail shape. `CultNetErrorMessageFormatter` owns + C# bytes. The refusal subclass owns the code-to-details binding. + - Inputs: the refusal instance, messageId, routingHint. + - Outputs: bytes, and `cs-written.json` for the other runtimes. + - Derived state: `CultNetErrorMessage.Code` becomes derived; the refusal decides it. + - Forbidden writers: any `Code = "..."` assignment (it cannot compile); `[Key]` attribute serialization + of the message; hand-typed hex anywhere. + - Shared paths: every server send uses the factories; every receiver uses `CultNetPeerErrorException`. + - Deletion line: `CultNetErrorDetails` and the settable `Code`. +- **Verification** (`dotnet` image): + - Builds: `tests/GameCult.Networking.Tests`, `tests/GameCult.Mesh.Tests`. + - `ErrorVectorTests` with `CULTNET_WRITE_VECTORS=1`, then without, and check that the committed file + reproduces byte-for-byte. + - Tests and the rules they pin: + - vector round trip: bytes, decode, bytes are equal for every entry; + - a registry factory is byte-equal to its vector; + - every vector validates against the schema; + - an unknown code decodes to `CultNetUnknownRefusal` with its raw details intact; + - a Python-shaped error (`py_snapshot_byte_limit` above, with the Q2/Q3 shape) decodes typed with + `responseBytes` and `maxSnapshotBytes` intact. Today's C# silently empties those. + - content-session correlation: two pending requests, an error for the second, and the first still + completes. + - Mutation: Stryker.NET on the formatter and factories. The survivors that must die: + - swapping two known codes whose details are identical (`snapshot_byte_limit_exceeded` and + `document_unservable`); + - padding a nil key into details; + - dropping `messageId` on encode; + - mapping an unknown code to null. + - Negative greps (checked at HEAD, where they hit the sites being cut): + - `git grep -nE 'new (IOException|InvalidOperationException)\(error\.Error\)' -- src` must return 0 + (9 hits at HEAD). + - `git grep -nE 'Field = "(schemaId|subscriptionId)"' -- src` must return 0 (2 at HEAD). + - Operator: none. +- **Ledger:** about -60 C# (the details record, the settable code, the probe printer, and the per-factory + schema tests), +250 (the formatter, the refusal types, the vector writer). The schema grows by about 90 + JSON lines. The net is positive, bought by Q1/Q2 typing in the reference. + +### Cut 2. Rust: cultnet-rs and cultmesh-rs + +- **Deletes first:** + - `CultNetErrorCode` and `CultNetErrorDetails` as separate types (`packages/cultnet-rs/src/contracts.rs:252-310`, + 59 lines); + - `require_error_details` (`:1121-1146`); + - `encode_error_details` (`:1339-1366`); + - the eight hex `Case`s and the header comment in `packages/cultnet-rs/tests/error_message.rs:1-160`. + That file is rewritten to read the vector file. + - Keep `require_legacy_optional_u64` (`:1149`). Check with `git grep` for other callers first; at HEAD, + only the error details use it. +- **Adds:** + - `CultNetMessage::Error { error: String, message_id: Option, routing_hint: Option, refusal: Option }`. + - `enum CultNetRefusal`, with one variant per registry code and its typed fields, plus + `Other { code: String, details: Option }`. + - `CultNetRefusal::code(&self) -> &str`. + - A constructor `CultNetMessage::error(text)` and builder-style `with_refusal` / `with_message_id`, so + consumers stop writing struct literals. Huginn has three. + - Decode reads every top-level key by name and routes unknown codes to `Other` (Q1-B). + - Encode writes the Cut 1 key order. + - `CultNetShardRoutingHint` is typed, reusing the crate's existing shard descriptor type. +- **Per-file changes:** + - `packages/cultnet-rs/src/contracts.rs:361-368` (variant), `:1097-1115` (decode), `:1300-1336` (encode). + - `packages/cultnet-rs/src/selection.rs:1210-1255`: the four refusals build `CultNetRefusal` variants. + - `packages/cultnet-rs/src/content.rs:316`: uses `CultNetMessage::error`. + - `packages/cultmesh-rs/src/lib.rs:812-827`: a peer error becomes a typed error carrying the refusal, + instead of `anyhow!` text. Name it `CultMeshPeerError`, next to the crate's existing error types. + - `packages/cultmesh-rs/src/rudp_document_server.rs` (Q4-A): each `CultMeshRudpRejectionReason` that + can be answered sends the matching coded error, correlated by the request's message id, before + returning `ApplicationRejected`. `PayloadBudgetFull` and `ResponseQueueFailed` stay local. +- **Authority map:** + - Owner: `CultNetRefusal` owns code and details. The schema owns the registry. + - Forbidden writers: string codes (`from_wire_str` survives only inside decode); struct literals of + details. + - Shared paths: selection refusals, document-server refusals and content errors all go through the one + encoder. +- **Verification** (`rust` image, `cd packages/cultnet-rs` then `cd ../cultmesh-rs`): + - `cargo test --workspace`. + - `tests/error_message.rs` pins: every vector decodes to `expected`; encoding the decoded value + reproduces the vector's bytes; the unknown-code vector decodes to `Other`; the routing vector keeps + `routing_hint`. + - cultmesh-rs: a put over the bound makes the peer receive `document_unservable` with the request's + message id, and details equal to the computed size and the bound. Run it at two bounds so that a + constant or an offset in `maxSnapshotBytes` dies. + - A peer that sends a Python-shaped `snapshot_byte_limit_exceeded` produces a `CultMeshPeerError`, not + a decode failure. + - Mutation: `cargo mutants --in-diff` on the cut's diff. Triage every survivor by name. + - Negative: `git grep -n 'CultNetErrorDetails' -- packages` must return 0. +- **Consumer note:** Huginn's `serve.rs:288/307/312` and its tests stop compiling at the next pin bump. + The migration is mechanical (`CultNetMessage::error(...)`), and `:312` adopts `unsupported_schema_version` + (Q7). Self routes it as a Huginn change; it is not part of this branch. + +### Cut 3. TypeScript: cultnet-ts, cultmesh-ts and cultmesh-browser + +- **Deletes first:** + - the two-field `CultNetErrorMessage` interface (`packages/cultnet-ts/src/contracts.ts:167-170`); + - the four literal error constructions in `packages/cultmesh-ts/src/index.ts:5875-5878`, `5885-5888`, + `5913-5916` and `5922-5925`, and the branch's codeless unservable site (`0eb7434a`); + - `rejectCorrelated`'s single-pending heuristic (`packages/cultmesh-browser/src/index.ts:724-731`). +- **Adds:** + - `CultNetErrorMessage` with every field. + - `type CultNetRefusal` as a discriminated union on `code`, with an `{ code: string; details: Record | null; known: false }` arm. + - `createCultNetError(text, { messageId, refusal })` as the only constructor. + - `class CultNetPeerError extends Error { message: CultNetErrorMessage }`. + - `parseCultNetMessage` keeps Ajv validation against the canonical schema; it now accepts every schema + branch, including unknown codes. +- **Per-file changes:** + - cultmesh-ts: the three "does not serve this" sites (`5875`, `5913`, `5922`) send + `unsupported_schema_version` (Q7-A); the snapshot catch (`5885`) sends a null code; the unservable site + sends `document_unservable` with details. + - cultmesh-ts `6527-6529` throws `CultNetPeerError`. + - cultmesh-browser `163`, `584` and `690`: correlate by `messageId`, falling back to today's behavior only + for a nil id. + - `packages/cultnet-ts/src/idunn-odin-presence-publisher.ts:118-120` throws `CultNetPeerError`. + - The legacy `gamecult.networking.v0` path (`contracts.ts:840-844`, `:899-900`) is unchanged. That is a + separate legacy contract. +- **Verification** (`kotlin` image, which carries Node 24; `npm install`, then build cultcache-ts before + cultnet-ts): + - A new `test/error-vectors.test.ts` pins that every vector parses and deep-equals `expected`, and that + `encode(createCultNetError(...))` is byte-equal to each registry vector. + - The cultmesh-ts document-server tests assert that each refusal parses with TypeScript's own + `parseCultNetMessage`. That is the probe's `ts_two_key` failure, pinned. + - Browser: two pending operations, an error for the second, and the first still resolves. + - Mutation: StrykerJS on the diff. + - Negative: `git grep -nE 'schemaVersion: "cultnet\.error\.v0",' -- packages/cultmesh-ts/src packages/cultmesh-browser/src` + must return 0. Exclude `cultnet-ts/src/schema-discovery.ts:302` (a descriptor) and + `contracts.ts:842` (legacy decode); both are legitimate hits for the unscoped pattern. + +### Cut 4. Python: cultnet-py and cultmesh-py + +- **Deletes first:** + - `_error_response` (`packages/cultmesh-py/src/cultmesh_py/server.py:573-589`); + - `unsupported_schema_version_error` (`packages/cultnet-py/src/cultnet_py/interop_peer.py:409-417`); + - every string literal code in `server.py` (lines in finding 3). +- **Adds:** + - `cultnet_py/errors.py`, holding: + - `CultNetErrorCode(StrEnum)`, the registry; + - a frozen `CultNetError` dataclass (`error`, `message_id`, `routing_hint`, `code: CultNetErrorCode | str | None`, + `details: Mapping | None`); + - one constructor function per code, taking exactly that code's keys; + - `to_wire()`, which writes the five top-level keys in the Cut 1 order, nil included; + - `parse_cultnet_error(dict)`. + - `CultNetPeerError` (`client.py:46-50`) keeps `.response` and gains `.error: CultNetError`. +- **Per-file changes:** + - Sites `server.py:158-163`, `355-394`, `555-569` and `593-598` and the branch's `document_unservable` + site use the constructors, with the Q5-A/Q6-A/Q7-A codes. + - `interop_peer.py` uses the constructor. + - `verify.py:335` reads `error.error.code`. + - Tests `packages/cultmesh-py/tests/test_cultmesh.py:2088`, `3791`, `4323` and `4363-4371`, and + `packages/cultnet-py/tests/test_cultnet.py:2342`, follow the renames. +- **Verification** (the image the campaign already uses for Python; state it in the report): + - A vector test pins that every vector parses to `expected` and that each constructor's `msgpack.packb` + is byte-equal to its vector. + - Mutation: mutmut on `errors.py` and the changed server sites. + - Negative: `git grep -nE '"schemaVersion": "cultnet\.error\.v0"' -- packages/cultnet-py/src packages/cultmesh-py/src` + returns only `errors.py`. +- **Not here:** F-1 (`schema_catalog.py` publishes its own shapes). + +### Cut 5. Kotlin: cultmesh-kotlin + +- **Adds:** + - A `CultNetErrorMessage` data class and a `sealed class CultNetRefusal` (known codes plus `Other`), + with a `CultNetMessage.asError(): CultNetErrorMessage?` view. + - `class CultNetPeerException(val error: CultNetErrorMessage) : IOException`. + - At `CultMesh.kt:1278` and `:1334`, and at every `receiveSchemaMessage` consumer that expects a + response, an error response throws `CultNetPeerException` before the `require`. +- **Verification** (`kotlin` image, `pwsh -File build.ps1`): + - A vector test pins that every vector decodes to `expected`. + - Kotlin emits no errors today, so encode parity applies only if a Kotlin server path is added. Say + so in the report; do not invent one. + - There is no mutation tool. Behavioural tests cover it: an error reply to a schema-catalog request + raises `CultNetPeerException` carrying the code. + +### Cut 6. Merge, release, consumers + +- Soul runs over the whole branch before the merge: + - rerun every vector suite; + - decode each runtime's live emissions with every other runtime. The interop lane from Cut 0 is the + harness; add an error round to `tests/GameCult.Networking.InteropPeer` and + `packages/cultnet-ts/test/interop` if Soul finds a runtime pair unreached. +- Release per `docs/semver-policy.md`, with a Soul pass before tags: + - `org.gamecult.cultlib` major, and the tracked Unity DLLs rebuilt, since they embed the schema; + - `cultnet-py` and `cultmesh-py` major. +- Consumers, routed by Self as separate repo changes: + - Huginn: pin bump plus Cut 2's note. + - StreamPixels and Heimdall: re-vendor. Heimdall is already broken by R-X. + - AetheriaEve, Gjallar and Mimir compile against the live CultLib path and read only `.Error`, so they + need no change. That is inferred from the audit; the next build of each confirms it. + - Hermodr: none. + +## 6. Subtraction ledger + +| Cut | Removed (estimate) | Added (estimate) | Formats / targets | +|---|---|---|---| +| 0 | 0 | 3 | none | +| 1 | ~60 C#, schema text | ~250 C#, ~90 schema | + `error-vectors.cs-written.json`; - `[Explicit]` hex printer | +| 2 | ~150 (types, two helpers, 160-line hex test) | ~200 | hex constants gone; vectors read from file | +| 3 | ~40 | ~120 | none | +| 4 | ~40 | ~120 (`errors.py`) | 8 ad-hoc codes → registry | +| 5 | 0 | ~80 | none | + +The net is positive, by roughly 500 lines, across five runtimes. It buys typed decode where there was +none (Kotlin, TS and Python), a correlation capability, and one vector file in place of hand-copied hex. +The pressure point: if Cut 1's formatter passes 150 lines, report it rather than trimming typing. + +## 7. Follow-ups outside this map + +- **F-1.** `packages/cultnet-py/src/cultnet_py/schema_catalog.py` publishes about 350 lines of hand-built + schemas under canonical `$id`s. Python should package and serve `contracts/cultnet/*.schema.json`, the + way Rust vendors them. It is a separate cut, because it covers every message, not errors. It can wait + because the catalog is advisory: no runtime validates a peer's message against a published descriptor. +- **F-2.** Codes for the remaining codeless refusals (authorization, failed snapshot sources, routing). + Wait for a consumer that branches on them. +- **F-3.** Whatever the Cut 0 lane reveals once its later steps run again. diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md index e7c8f6ae..1b519484 100644 --- a/docs/cultnet-rudp-ack-cut.md +++ b/docs/cultnet-rudp-ack-cut.md @@ -1,5 +1,28 @@ # CultNet RUDP: Acknowledged means delivered, one watermark owns ordered delivery, and a write belongs to its session +**PIVOT, 2026-09-30 (operator): RUDP is retired in favour of QUIC.** Operator: "RUDP really ballooned into +maintaining a robust custom transport across runtimes, when it started as just 'hey, let's do LiteNetLib but +better'. Is there any point maintaining it when we have QUIC?" and then "Yeah, pivot to QUIC adoption, I ain't +gonna let sunk costs take the wheel." +- **Landed 2026-09-30: Cut D, merged at `2f06ef84`** (Soul passes 1-4, batch 5 closing pass 4). It fixes live crashes: a spoofed datagram crashing Node, a malformed frame + killing Python's server, C# stale peers corrupting sessions. +- **Parked, not built:** Cut F, Cut 2 (held on F), Cut 4, Cut K and K1 (including the Kotlin resend-loop gap), Cut 7 + and the receipts port. Nothing of theirs exists to tag, so this map is their record. +- Defects found in RUDP from here on are fixed only if they are live crashes or data loss, until the QUIC campaign + sets RUDP's retirement. +- **Data-loss item from Odin Soul S-5 (2026-09-30).** At Odin's pin `3bf1c0ce`, cultnet-rs records a reliable + packet as received when it arrives, and `create_ack` reports it while ordered delivery still holds an earlier + packet. + - Probe: two pipelined puts, with the first datagram dropped. The second put was ACKed but not on disk. If the + first never arrives, the ACKed second put is never delivered. + - Sessions that carry a single put are safe. + - **Checked (Odin Q5 Imagination, probe P1 at `e4e2a0b9`): still open after ack Cuts 1-1c.** A held, undelivered + put is ACKed, so the pin bump does not close it, and it is a data-loss fix under the pivot rule. Related, P2: a + Pong ACKs a put received before the Ping; P3: a reliable reply created after a put ACKs it. The Odin writer-thread + cut (C5) adds `acknowledgement_withheld` to the document server; the transport fix belongs in cultnet-rs. +- The "CultNet over QUIC" campaign is new, so it opens as a typed Eureka campaign in the Huginn-backed session, not + as a prose map here. + Status: cut map, Imagination pass 3, 2026-09-30. Pass 3 maps the four rulings below, re-maps Cuts 1c and 4, and adds Cut F (FORWARD-TSN), Cuts 6a-6c (receipts), Cut 7 (expiry, if Q-A8), Cut K (Kotlin), Cut 2c (the C# fragment bound) and Cut D (a failed send to one peer stays that peer's loss). The text of Cuts 1, 1b, 2 @@ -730,7 +753,7 @@ The ack cuts build on these behaviours and do not alter them: | Rust hub `CultNetRudpServerHub` | `poll_resends` returns on the first failure (`rudp.rs:1762-1777`); the receive path's sends to one peer abort `receive_event_once` (`:1644`, `:1673`, `:1708`, `:1711`, `:1714`, `:1759`, through `send_packet` `:1799-1804`) | Cut D | | C# `CultNetRudpSocketTransportServer` (and `RudpCultNetSchemaServer` over it) | `PollResends` walks every peer and the first `SocketException` from `SendTo` (`CultNetTransport.cs:2829-2834`) escapes the loop (`:2796-2805`); `TryReceiveOnce`'s sends are unguarded except the refusal goodbye (`:2687`, `:2704`, `:2749-2750`, `:2788`; guarded `:2739-2745`) | Cut D | | cultmesh-py server | worse: an `OSError` from `sendto` (`server.py:302-307`) in `_poll_rudp_resends` (`:277-285`) or the receive path escapes `_rudp_loop` (`:203-…`, which catches only `recvfrom`'s errors), and the RUDP thread dies for every peer | Cut D | -| cultmesh-ts document server | none: Node's `socket.send` is asynchronous and reports failures on the socket's `error` event (`cultmesh-ts/src/index.ts:5858`), which `reportError` logs (`:5778-5785`) | none | +| cultmesh-ts document server | none. **Corrected 2026-09-30:** Node does not report send failures on the socket's `error` event. With a callback they go to the callback; without one, libuv drops them silently (Node 24). Only port 0 throws synchronously. Cut D routes every send through `sendRudpDatagram`. | none | | Kotlin interop server | none that matters: a catch-all logs and continues (K:4025), and it is interop-only | none (Cut K1 reviews it) | @@ -1048,8 +1071,8 @@ Design in §10. Independent of every other cut; it touches send sites, not sessi (`:2739-2745`) becomes redundant and goes; - cultmesh-py `server.py`: `_send_rudp_packet` (`:302-307`) catches `OSError` and counts it. - **Not changed:** single-peer socket transports (a failed send already concerns only their one peer, and - their callers, flush among them, get the error); `cultmesh-ts`'s document server (Node reports send - failures on the socket's `error` event, `index.ts:5858`). + their callers, flush among them, get the error); `cultmesh-ts`'s document server. (Corrected 2026-09-30: Node reports send failures only to a send + callback, never on the socket's `error` event; see the table above.) - **Authority map:** - Owner: each peer's session owns its peer's fate (idle timeout, lifetime, refusal, Disconnect). A send failure is an input to nothing but a counter. @@ -1590,6 +1613,25 @@ profiles. a permanent failure is returned from a caller-directed send and, inside a poll, ends only that peer's session with a typed reason; transient failures stay counted losses. Windows `ConnectionResetError` on receive is idle, never fatal, in every server. Cut D's negative grep `send_packet(...)?;` is obsolete: the `?` now propagates encode errors only. +- **Cut 3 merged at `1210fdc7` (2026-09-30).** + - Batch 2 fixed Rust `send`, which checked the window before purging expired packets. It also added a + new-generation test in all four runtimes. + - Soul pass on batch 2: nothing found. 18 mutants were killed. Interop was 27/27, run in `ack1d-interop:2`, + whose Linux wrappers let the Windows-only Kotlin build run. + - Residual: `packages/cultmesh-rs/Cargo.lock` is stale. + - Also recorded: `packages/cultmesh-kotlin/build.ps1` is Windows-only, so Kotlin interop runs off Windows only + through an image with wrappers. + - P2 (the expiry hole) stays with Cut F. +- **Cut D batch 2, Soul pass 2: fix first.** + - cultnet-ts emits transient send failures as `"error"`, which crashes processes with no listener. Section 10's + claim that "Node reports send failures on the socket's error event" is false on Node 24 without a callback: + they are dropped silently. + - C# stale peer objects write into the replacing session. + - cultmesh-py dies on a non-ValueError per-peer failure. + - Six guards are unpinned. + - Batch 3 is in Hands. + - **K1 gains:** Kotlin's idle resend loop (`CultMesh.kt:4033-4036`) is one `try`, so one peer's permanent failure + stops resends for every later peer. That is in addition to a refused packet not ending that peer's session. - **Recorded (liveness, not fixed):** a lowest reliable packet lost forever stalls the sender 1,023 sequences ahead while the peer stays alive; session timeout doesn't end it. TCP ends such a connection after a retransmission limit (R2). Decide with Cut F (abandonment) whether a reliable packet outstanding past a bound ends the session. diff --git a/docs/cultnet-transport-parity.md b/docs/cultnet-transport-parity.md index d0779196..17fc6964 100644 --- a/docs/cultnet-transport-parity.md +++ b/docs/cultnet-transport-parity.md @@ -271,6 +271,15 @@ Current progress: flush where the runtime supports one, so callers that require confirmed transport delivery do not substitute a fixed sleep. TCP-framed profiles do not publish the field because they do not own the RUDP pending-reliable queue. +- The receive window is a contract, not a discovery. A receiver must hold 1,023 + sequences and 4 MiB of payload beyond its contiguous received watermark, and + a conforming sender never exceeds either. The sender's flow window enforces + it without a wire field: a reliable packet is admitted only while its + sequence is at most 1,023 above the lowest unacked sequence and the payload + above that sequence stays within 4 MiB. The first packet is always + admissible, so a payload larger than 4 MiB goes out alone, and packets wait + in the FIFO in order, so a small packet never overtakes a large one. C#, + Rust, TypeScript, and Python enforce it; Kotlin does not yet. - TypeScript, C#, Rust, Python, and Kotlin now share a portable `cultnet.reconnect_policy.v0` document and deterministic exponential delay helper, and RUDP transport profiles advertise that policy under diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index 5e191df8..bd340c87 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -369,6 +369,34 @@ stripped (probe). Other findings: refuses v3 also contains C2a, so no released reader strips ids. C0 and C1 are never released on their own. Raw and cross-runtime writers that cannot see ids stay sticky-only. Fix batch 4 is in Hands. +**C2a F2 ruled (operator, 2026-09-30, accepting Self's recommendation B): records nobody claims are carried +untouched.** A store may hold records under a schema id that no local type owns or lists as compatible, for +example after a rename that declared nothing. +- Both store kinds carry those records byte for byte under their own id. The catalog lists them as foreign + records. +- Writes to other records proceed. +- A write that would overwrite or relabel an unclaimed record is refused with a typed schema conflict. +- The single-file store's whole-view write stops relabelling such records onto local types. The directory + store stops refusing every commit while it holds one. +- Why: batches 8-9 showed that the danger is rewriting a record under a new identity (F1, F6), not an + undeclared old id. Batch 10's owner-beats-lister rule makes declaring an old id cheap anyway. +- Self's first lean, "refuse loudly" (A), is superseded. + +**C2a batches 9-10 (Soul pass on `fdb8a60a`, 2026-09-30): hold.** +- **F6, A-B-A lost update on the single-file store.** Ruled by Self: any write that changes a record's bytes or + schema id mints a later `storedAt`. +- **F8, batch 9 over-refused.** An owner beats a lister, so v1 and v2 coexist. +- **F9.** Compatible ids compare as a set. +- **F10.** Typed refusal errors in TS and Python. +- Batch 10 is in Hands. +- **F7 ruled (operator, 2026-09-30): retire `AsSchemaAlias`.** "Schema aliasing sounds like a really bad idea, I + support retiring it." + - C# refuses a second type for one schema, as TS, Python and Rust already do. There is one type per schema in + every runtime. + - The API, its registry path (`IsExactWireAlias`) and the tests that exercised it (7 Mesh, 2 Caching) are + deleted or rewritten. + - No consumer called it. + **Operator rulings, 2026-09-30:** - **Duplicate unique-index values do not load** ("Duplicate indices should not load"). This covers plain records too, so R6-for-indexes becomes one general rule: diff --git a/docs/nuget-packaging.md b/docs/nuget-packaging.md index 1da3ba63..ed8746bb 100644 --- a/docs/nuget-packaging.md +++ b/docs/nuget-packaging.md @@ -11,9 +11,12 @@ CultNet, and CultCache ownership boundaries. powershell -ExecutionPolicy Bypass -File .\scripts\pack-nuget.ps1 ``` -The command creates a local feed under `artifacts/nuget`, verifies that every -internal dependency package exists at the same version, then restores and runs -a clean .NET consumer using only `PackageReference Include="GameCult.Mesh"`. +The command packs `GameCult.Mesh`, `GameCult.Networking.WebSockets`, and every +project they reach through `ProjectReference` (including `GameCult.Math`, which +carries its own version) into a local feed under `artifacts/nuget`. It verifies +that each of those packages exists at the identity MSBuild reports for it, then +restores and runs a clean .NET consumer using only +`PackageReference Include="GameCult.Mesh"`. The Unity distribution remains `org.gamecult.cultlib`, built by `scripts/build-unity-package.ps1`, because Unity consumes the managed assembly diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 2117c2d8..368b5338 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -11,6 +11,7 @@ import java.io.InputStream import java.io.OutputStream import java.net.DatagramPacket import java.net.DatagramSocket +import java.net.Inet6Address import java.net.InetAddress import java.net.InetSocketAddress import java.net.ServerSocket @@ -581,7 +582,7 @@ object CultMesh { connectionId: Long, remoteHost: String, remotePort: Int, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = cultNetRudpClient( @@ -600,7 +601,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: CultNetRudpEndpoint, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = createRudpClient( @@ -617,7 +618,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: String, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = createRudpClient( @@ -633,7 +634,7 @@ object CultMesh { runtimeId: String, connectionId: Long, peer: CultMeshPeerCard, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { @@ -651,7 +652,7 @@ object CultMesh { role: String, shardId: String? = null, at: Instant = Instant.now(), - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { @@ -664,7 +665,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: CultNetRudpEndpoint, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -683,7 +684,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: String, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -705,7 +706,7 @@ object CultMesh { runtimeId: String, connectionId: Long, peer: CultMeshPeerCard, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -736,7 +737,7 @@ object CultMesh { role: String, shardId: String? = null, at: Instant = Instant.now(), - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -3028,22 +3029,36 @@ fun cultNetRudpServer( ) } +/** + * The address an RUDP client binds when its caller names none: loopback for a + * loopback endpoint, otherwise the unspecified address of the endpoint's family. + * A socket bound to loopback cannot send off the host on Windows. + */ +fun cultNetRudpClientBindAddress(remote: InetAddress): InetAddress = when { + remote is Inet6Address && remote.isLoopbackAddress -> InetAddress.getByName("::1") + remote is Inet6Address -> InetAddress.getByName("::") + remote.isLoopbackAddress -> InetAddress.getByName("127.0.0.1") + else -> InetAddress.getByName("0.0.0.0") +} + fun cultNetRudpClient( runtimeId: String, connectionId: Long, remoteHost: String, remotePort: Int, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { - val socket = DatagramSocket(bindPort, InetAddress.getByName(bindHost)).also { it.soTimeout = 20 } + val remote = InetAddress.getByName(remoteHost) + val bindAddress = bindHost?.let(InetAddress::getByName) ?: cultNetRudpClientBindAddress(remote) + val socket = DatagramSocket(bindPort, bindAddress).also { it.soTimeout = 20 } return CultNetRudpSocketTransportConnection( socket = socket, mode = CultNetRudpSocketMode.Client, runtimeId = runtimeId, connectionId = connectionId, - remoteAddress = InetSocketAddress(InetAddress.getByName(remoteHost), remotePort), + remoteAddress = InetSocketAddress(remote, remotePort), initialSequence = tuning.initialSequence, resendDelayMs = tuning.resendDelayMs, maxFragmentBytes = tuning.maxFragmentBytes, @@ -3368,6 +3383,7 @@ fun main(args: Array) { rudpSessionFragmentsAndReassemblesReliableOrderedPayloads() rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() + rudpClientBindsLoopbackOnlyForALoopbackEndpoint() rudpSocketTransportHandshakesAndCarriesReliableOrderedSchemaFrames() rudpSocketTransportCarriesFragmentedReliableOrderedSchemaFrames() return @@ -5378,6 +5394,27 @@ private fun rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() { check(oldAck.ackMask == 0L) } +private fun rudpClientBoundHost(endpoint: String, bindHost: String? = null): InetAddress = + CultMesh.createRudpClient("kotlin-rudp-bind-client", 0x10203060L, endpoint, bindHost).use { client -> + InetAddress.getByName(client.profile.transports.single().host) + } + +// A client bound to loopback cannot send to another host on Windows, so a +// client for a remote endpoint binds the unspecified address. A JVM wildcard +// socket is dual-stack and reports one wildcard for either family, so the +// family is not observable here. +private fun rudpClientBindsLoopbackOnlyForALoopbackEndpoint() { + check(rudpClientBoundHost("rudp://10.77.0.1:17872").isAnyLocalAddress) + check(rudpClientBoundHost("rudp://[2001:db8::1]:17872").isAnyLocalAddress) + check(rudpClientBoundHost("rudp://127.0.0.1:17872") == InetAddress.getByName("127.0.0.1")) + check(rudpClientBoundHost("rudp://[::1]:17872") == InetAddress.getByName("::1")) + check(rudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1") == InetAddress.getByName("127.0.0.1")) + check(rudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0").isAnyLocalAddress) + CultMesh.createRudpServer("kotlin-rudp-bind-server", 0x10203061L).use { server -> + check(InetAddress.getByName(server.profile.transports.single().host) == InetAddress.getByName("127.0.0.1")) + } +} + private fun rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() { val connectionId = 0x10203042L CultMesh.createRudpServer( diff --git a/packages/cultmesh-py/src/cultmesh_py/facade.py b/packages/cultmesh-py/src/cultmesh_py/facade.py index c6b781c2..84bea2c8 100644 --- a/packages/cultmesh-py/src/cultmesh_py/facade.py +++ b/packages/cultmesh-py/src/cultmesh_py/facade.py @@ -18,6 +18,7 @@ from cultnet_py import ( CultNetShardCatalog, CultNetSimulationObservationHub, create_rudp_schema_transport, + rudp_client_bind_host, wire_message_schema_catalog, ) @@ -188,7 +189,7 @@ class CultMesh: connection_id: int, endpoint: str | CultMeshRudpEndpoint, *, - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, socket: socket_module.socket | None = None, initial_sequence: int | None = None, @@ -204,7 +205,10 @@ class CultMesh: if isinstance(endpoint, str) else endpoint ) - transport_socket = socket or _bind_rudp_socket(bind_host, bind_port) + transport_socket = socket or _bind_rudp_socket( + bind_host if bind_host is not None else rudp_client_bind_host(parsed_endpoint.host), + bind_port, + ) return CultNetRudpSocketTransportConnection( CultNetRudpSocketTransportOptions( runtime_id=runtime_id, @@ -433,7 +437,7 @@ class CultMesh: endpoint: str | CultMeshRudpEndpoint | None = None, connection_id: int = 0x43554C54, runtime_id: str = "cultmesh-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, ) -> CultNetRawClient: if endpoint is None and host.lower().startswith("rudp://"): @@ -469,7 +473,7 @@ class CultMesh: endpoint: str | CultMeshRudpEndpoint | None = None, connection_id: int = 0x43554C54, runtime_id: str = "cultmesh-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, ) -> CultNetRawClient: return CultMesh.create_client( @@ -485,7 +489,10 @@ class CultMesh: def _bind_rudp_socket(bind_host: str, bind_port: int) -> socket_module.socket: - transport_socket = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_DGRAM) + transport_socket = socket_module.socket( + socket_module.AF_INET6 if ":" in bind_host else socket_module.AF_INET, + socket_module.SOCK_DGRAM, + ) transport_socket.bind((bind_host, bind_port)) transport_socket.settimeout(0.02) return transport_socket diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index f6453d23..d9f4c349 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -22,6 +22,7 @@ from cultnet_py import ( create_tcp_framed_transport_profile, decode_rudp_packet, encode_rudp_packet, + is_permanent_send_error, wire_message_schema_descriptors, ) @@ -35,6 +36,9 @@ _MAX_FRAGMENTS_PER_MESSAGE = 0xFFFF # Replies larger than one fragment are split into fragments of at most this many bytes, # as the Rust and TypeScript document servers do by default. CULTMESH_RUDP_DEFAULT_MAX_FRAGMENT_BYTES = 1200 +# The answer to a request whose reply the peer's reliable queue cannot take. Fixed text: it +# never quotes the reply or the queue's error. +_REPLY_UNQUEUEABLE = "The reply could not be queued for this peer." from cultnet_py.cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, VERSE_CATALOG_REQUEST, @@ -96,9 +100,11 @@ class CultMeshLocalServer: @property def rudp_send_failures(self) -> int: - """RUDP replies and datagrams that could not be sent to a peer. Each is that peer's - loss: a reliable packet already queued stays pending and is resent, and no other - peer is affected.""" + """Datagrams that could not be sent to a peer. Each is that peer's lost + datagram: a reliable packet stays pending and is resent, and no other + peer is affected. A datagram that can never be sent as built + (`is_permanent_send_error`) is not counted here: it ends that peer's + session, and the peer's goodbye names the error.""" return self._rudp_send_failures def __post_init__(self) -> None: @@ -265,7 +271,10 @@ class CultMeshLocalServer: while not self._stop.is_set(): try: wire, remote_addr = rudp_socket.recvfrom(65535) - except TimeoutError: + except (TimeoutError, ConnectionResetError): + # Windows reports an earlier ICMP port-unreachable, from a peer that + # has gone away, as an error on the next receive. It names no datagram + # of this read, so it is idle, not the end of the thread. self._poll_rudp_resends(rudp_socket, peers) continue except OSError: @@ -295,46 +304,75 @@ class CultMeshLocalServer: ) ) peers[remote_addr] = peer - self._send_rudp_packet( + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.accept_connect(packet, now_ms, b"cultmesh-local-rudp"), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) continue if peer is None: continue - result = peer.session.receive(packet, now_ms) - if result.reply is not None: - self._send_rudp_packet(rudp_socket, remote_addr, result.reply) - for ready in result.ready_to_send: - self._send_rudp_packet(rudp_socket, remote_addr, ready) + try: + result = peer.session.receive(packet, now_ms) + except ValueError: + # receive() has already recorded the packet's reliable sequence, so the + # session cannot be kept: a retransmit would be acknowledged and the + # frame silently lost. It ends, never the thread, and the peer is told. + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue + unsendable: OSError | None = None + for outgoing in (*([result.reply] if result.reply is not None else []), *result.ready_to_send): + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, outgoing) + if unsendable is not None: + break if result.disconnected: peers.pop(remote_addr, None) continue - for delivered in result.delivered: - if delivered.channel_id != "schema": - continue - try: - message = msgpack.unpackb(delivered.payload, raw=False) - except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): - continue - if not isinstance(message, dict): - continue - responses = self._handle_connection_message(message, peer.subscriptions, over_rudp=True) - for response in responses: - self._send_rudp_schema_frame( - rudp_socket, - remote_addr, - peer.session, - msgpack.packb(response, use_bin_type=True), - ) - if packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: - self._send_rudp_packet( + try: + for delivered in result.delivered: + if delivered.channel_id != "schema": + continue + try: + message = msgpack.unpackb(delivered.payload, raw=False) + except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): + continue + if not isinstance(message, dict): + continue + responses = self._handle_connection_message(message, peer.subscriptions, over_rudp=True) + for response in responses: + if unsendable is None: + unsendable = self._send_rudp_schema_frame( + rudp_socket, + remote_addr, + peer.session, + msgpack.packb(response, use_bin_type=True), + message_id=str(response.get("messageId") or ""), + ) + except Exception as error: # noqa: BLE001 - one peer's message never ends the thread + # Whatever a peer's message makes the handling raise is that peer's failure. + # The session recorded the frame's sequence, so it cannot be kept: a retransmit + # would be acknowledged and never handled. The session ends, never the thread, + # and the peer is told. The log names the error type only, because the error's + # own message can quote what the peer sent. + _LOGGER.warning( + "CultMesh RUDP server could not handle a peer's message (%s); that peer's session ended.", + type(error).__name__, + ) + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue + if unsendable is None and ( + packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered + ): + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.create_ack_for_received(packet.sequence), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) def _poll_rudp_resends( self, @@ -344,7 +382,10 @@ class CultMeshLocalServer: now_ms = _now_ms() for remote_addr, peer in list(peers.items()): for packet in peer.session.due_resends(now_ms): - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) + break def _send_rudp_schema_frame( self, @@ -352,7 +393,9 @@ class CultMeshLocalServer: remote_addr: tuple[str, int], session: CultNetRudpSession, payload: bytes, - ) -> None: + *, + message_id: str = "", + ) -> OSError | None: try: packets = session.send_many( "schema", @@ -361,25 +404,75 @@ class CultMeshLocalServer: max_fragment_bytes=self.rudp_max_fragment_bytes, ) except ValueError: - # The peer's reliable queue cannot take this reply now. It is that peer's loss, - # never the thread's. - self._rudp_send_failures += 1 - return + # The peer's reliable queue cannot take this reply. The peer is answered, not left + # to time out: a small cultnet.error.v0 goes unreliable and unordered, so it needs + # no room in the queue that just refused. Then the error ends the session, as any + # message the handling cannot serve does. + refusal = self._error_response( + _REPLY_UNQUEUEABLE, + message_id=message_id, + code="response_unqueueable", + ) + for packet in session.send_many( + "schema", + msgpack.packb(refusal, use_bin_type=True), + CultNetRudpSendOptions(reliable=False, ordered=False, now_ms=_now_ms()), + max_fragment_bytes=self.rudp_max_fragment_bytes, + ): + if self._send_rudp_packet(rudp_socket, remote_addr, packet) is not None: + break + raise for packet in packets: - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + return unsendable + return None def _send_rudp_packet( self, rudp_socket: socket.socket, remote_addr: tuple[str, int], packet: CultNetRudpPacket, - ) -> None: - # A datagram that cannot leave is that peer's lost datagram: counted, never raised - # (raising here ended the RUDP thread for every peer). A reliable one is resent. + ) -> OSError | None: + # A transient send failure is that peer's lost datagram: counted, never + # raised (raising here ended the RUDP thread for every peer). A permanent + # one (`is_permanent_send_error`) is returned: the datagram can never be + # sent as built, and the caller ends that peer's session. + wire = encode_rudp_packet(packet) try: - rudp_socket.sendto(encode_rudp_packet(packet), remote_addr) - except OSError: + rudp_socket.sendto(wire, remote_addr) + except OSError as error: + if is_permanent_send_error(error): + return error self._rudp_send_failures += 1 + return None + + def _end_refused_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], + ) -> None: + peer = peers.pop(remote_addr) + # The goodbye is built after the reset, or its ack field would acknowledge the + # very frame the session refused. + peer.session.reset_peer_state() + goodbye = peer.session.create_disconnect(b"session refused a packet") + self._send_rudp_packet(rudp_socket, remote_addr, goodbye) + + def _end_unsendable_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], + error: OSError, + ) -> None: + """The session owes its peer a packet that can never be sent as built. It ends, + never the thread, and the peer's goodbye names the error.""" + peer = peers.pop(remote_addr, None) + if peer is None: + return + self._send_rudp_packet(rudp_socket, remote_addr, peer.session.end_unsendable_session(error)) def _handle_connection_message( self, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index cb4b767d..18e13b0b 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import json import os import socket @@ -49,11 +50,13 @@ from cultnet_py import ( CultNetSimulationConsensusCandidate, CultNetSimulationObservation, CultNetSimulationObservationHub, + CultNetRudpPacketType, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, CultNetRudpSocketTransportOptions, apply_shard_log_response, create_rudp_schema_transport, + decode_rudp_packet, document_delete, document_put_raw, hello, @@ -216,6 +219,37 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(received_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") self.assertEqual(received_messages[0]["messageId"], "rudp-transport-factory") + def _rudp_client_bound_host(self, endpoint: str, bind_host: str | None = None) -> str: + client = CultMesh.create_rudp_client("python-cultmesh-bind-client", 0x10203060, endpoint, bind_host=bind_host) + try: + return client.socket.getsockname()[0] + finally: + client.socket.close() + + # A client bound to loopback cannot send to another host on Windows, so a + # client for a remote endpoint binds the unspecified address of its family. + def test_cultmesh_rudp_client_for_a_remote_endpoint_binds_the_unspecified_address(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872"), "0.0.0.0") + self.assertEqual(self._rudp_client_bound_host("rudp://[2001:db8::1]:17872"), "::") + + def test_cultmesh_rudp_client_for_a_loopback_endpoint_binds_loopback(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872"), "127.0.0.1") + self.assertEqual(self._rudp_client_bound_host("rudp://[::1]:17872"), "::1") + + def test_cultmesh_rudp_client_binds_an_explicit_bind_host_whatever_the_endpoint(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872", "127.0.0.1"), "127.0.0.1") + self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872", "0.0.0.0"), "0.0.0.0") + + def test_cultmesh_rudp_server_binds_loopback_unless_told_otherwise(self) -> None: + loopback = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203061) + anywhere = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203062, bind_host="0.0.0.0") + try: + self.assertEqual(loopback.socket.getsockname()[0], "127.0.0.1") + self.assertEqual(anywhere.socket.getsockname()[0], "0.0.0.0") + finally: + loopback.socket.close() + anywhere.socket.close() + def test_cultmesh_facade_creates_rudp_client_from_peer_endpoint(self) -> None: connection_id = 0x10203042 server = CultMesh.create_rudp_server( @@ -4456,8 +4490,9 @@ class CultMeshTests(unittest.TestCase): def test_cultmesh_local_server_refuses_a_rudp_put_whose_reply_would_overflow_the_reliable_queue(self) -> None: # Replies go in fragments of 100 bytes and a peer's reliable queue holds 8: a put over RUDP # whose served reply takes 8 fragments is admitted and served, 9 or 20 are refused. A - # snapshot of two such documents needs 16 and cannot be queued: that peer's loss, counted, - # and the thread serves on. + # snapshot of two such documents needs 16 and cannot be queued: the peer is answered with a + # small cultnet.error.v0 rather than left to time out, its session ends, and the thread + # serves on. document = self._rudp_bound_note() body_for = self._rudp_body_for(document, "note:q") @@ -4475,20 +4510,11 @@ class CultMeshTests(unittest.TestCase): self._rudp_note_put(document, "put-at", "note:q", body_for(800)), snapshot_request(message_id="").to_wire(), ) - failures_before = server.rudp_send_failures - transport = create_rudp_schema_transport( - host="127.0.0.1", port=server.port, connection_id=server.rudp_connection_id, - timeout_seconds=4.0, max_fragment_bytes=1200, + unqueueable, _ = self._rudp_exchange( + server, + self._rudp_note_put(document, "put-r", "note:r", body_for(800)), + snapshot_request(message_id="both").to_wire(), ) - try: - transport.send("schema", msgpack.packb(self._rudp_note_put(document, "put-r", "note:r", body_for(800)), use_bin_type=True)) - transport.send("schema", msgpack.packb(snapshot_request(message_id="both").to_wire(), use_bin_type=True)) - deadline = time.monotonic() + 4.0 - while server.rudp_send_failures == failures_before and time.monotonic() < deadline: - transport.receive_once() - time.sleep(0.005) - finally: - transport.close() failures_after = server.rudp_send_failures # An empty snapshot fits the small queue; a hello reply would not. after_reply, _ = self._rudp_exchange( @@ -4514,8 +4540,10 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(served["schemaVersion"], "cultnet.snapshot_response_raw.v0") self.assertEqual([record["recordKey"] for record in served["documents"]], ["note:q"]) self.assertEqual(len(served_payload), 800) - self.assertEqual(failures_before, 0) - self.assertEqual(failures_after, 1) + self.assertEqual(unqueueable["schemaVersion"], "cultnet.error.v0") + self.assertEqual(unqueueable["messageId"], "both") + self.assertEqual(unqueueable["code"], "response_unqueueable") + self.assertEqual(failures_after, 0) self.assertEqual(after_reply["messageId"], "after-queue") self.assertTrue(alive) @@ -4551,8 +4579,9 @@ class CultMeshTests(unittest.TestCase): self.assertIsNotNone(stored) def test_cultmesh_local_server_rudp_thread_survives_a_datagram_that_cannot_leave(self) -> None: - # Fragments larger than any UDP datagram: the reply's one packet cannot be sent. That - # peer's loss is counted; the thread serves the next peer. + # Fragments larger than any UDP datagram: the reply's one packet can never be sent as + # built. That is a permanent failure: it ends that peer's session and is not counted as + # a lost datagram; the thread serves the next peer. document = self._rudp_bound_note() node = CultMesh.create_node(runtime_id="mesh-rudp-emsgsize") node.database.register_document(document) @@ -4565,9 +4594,12 @@ class CultMeshTests(unittest.TestCase): try: transport.send("schema", msgpack.packb(self._rudp_note_put(document, "put-large", "note:big", 80_000), use_bin_type=True)) transport.send("schema", msgpack.packb(snapshot_request(message_id="big").to_wire(), use_bin_type=True)) - deadline = time.monotonic() + 4.0 - while server.rudp_send_failures == 0 and time.monotonic() < deadline: - transport.receive_once() + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + try: + transport.receive_once() + except ConnectionError: + break time.sleep(0.005) finally: transport.close() @@ -4576,7 +4608,7 @@ class CultMeshTests(unittest.TestCase): alive = server._rudp_thread is not None and server._rudp_thread.is_alive() finally: server.stop() - self.assertGreaterEqual(failures, 1) + self.assertEqual(failures, 0) self.assertEqual(hello_reply["schemaVersion"], "cultnet.hello.v0") self.assertTrue(alive) @@ -4948,5 +4980,459 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42) +class CultMeshRudpSendFailureTests(unittest.TestCase): + """One peer whose datagrams cannot be sent must not stop the RUDP thread + serving the others.""" + + def _client(self, server: Any, name: str) -> tuple[CultNetRudpSocketTransportConnection, tuple[str, int]]: + sock = bind_udp_socket() + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id=name, + socket=sock, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + client.connect(b"join") + for _ in range(200): + client.receive_once() + if client.connected: + break + self.assertTrue(client.connected) + return client, sock.getsockname()[:2] + + def test_a_failed_send_to_one_peer_does_not_kill_the_rudp_thread(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + raise OSError("injected send failure") + return original_sendto(sock, data, *args) + + request = hello(runtime_id="probe").to_bytes() + with patch.object(socket.socket, "sendto", sendto): + # The server's reply and ack to X both fail to send. + x.send("schema", request) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertTrue(server._rudp_thread.is_alive()) + + y.send("schema", request) + response = None + for _ in range(200): + response = y.receive_once() + if response is not None: + break + time.sleep(0.005) + + self.assertIsNotNone(response, "Y is still served after X's send failed") + self.assertEqual(msgpack.unpackb(response.payload, raw=False)["runtimeId"], "mesh-server") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertGreater(server.rudp_send_failures, 0) + finally: + server.stop() + + def _served(self, y: CultNetRudpSocketTransportConnection) -> Any: + import msgpack # type: ignore + + y.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + response = y.receive_once() + if response is not None: + return msgpack.unpackb(response.payload, raw=False) + time.sleep(0.005) + return None + + def test_a_vanished_peer_does_not_kill_the_rudp_thread(self) -> None: + # On Windows the closed port of a peer that went away is reported as + # ConnectionResetError (WinError 10054) by the next receive. It can only fail there. + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + x.send("schema", hello(runtime_id="probe").to_bytes()) + x.socket.close() + time.sleep(1.0) # the server's reply and resends to X draw ICMP port-unreachable + + self.assertTrue(server._rudp_thread.is_alive()) + served = self._served(y) + self.assertIsNotNone(served, "Y is still served after X vanished") + self.assertEqual(served["runtimeId"], "mesh-server") + finally: + server.stop() + + def test_a_reset_reported_by_a_receive_is_idle_not_the_end_of_the_thread(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_recvfrom = socket.socket.recvfrom + resets = [3] + try: + y, _ = self._client(server, "peer-y") + + def recvfrom(sock: socket.socket, *args: Any) -> Any: + if sock is server._rudp_socket and resets[0] > 0: + resets[0] -= 1 + raise ConnectionResetError(errno.ECONNRESET, "injected reset") + return original_recvfrom(sock, *args) + + with patch.object(socket.socket, "recvfrom", recvfrom): + self._wait_for(lambda: resets[0] == 0) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_send_failure_ends_only_that_peers_session_and_the_goodbye_names_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The server's response to X can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason, "X is told its session ended") + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertGreater(len(x.disconnect_reason), len(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + + # Y is unaffected. + self.assertFalse(x.connected) + self.assertIsNotNone(self._served(y)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + + def test_healthy_sends_are_not_failures(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + self.assertIsNotNone(self._served(x)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + + def test_a_permanent_failure_in_a_resend_ends_only_that_peers_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + mode = ["transient"] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # X's response is lost, so a resend is pending; then it can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertFalse(x.disconnect_reason, "a transient failure ends no session") + mode[0] = "permanent" + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_failure_acknowledging_a_frame_ends_the_session(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # Not a message the server answers, so its ack is the only datagram owed. + x.send("schema", msgpack.packb(1)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_connect_whose_accept_can_never_be_sent_starts_no_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + y, _ = self._client(server, "peer-y") + z_socket = bind_udp_socket() + z_addr = z_socket.getsockname()[:2] + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(z_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + z = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="peer-z", + socket=z_socket, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + z.connect(b"join") + # A session that started would resend its Accept, and the resend would get through. + time.sleep(0.5) + z_socket.settimeout(0.05) + received = [] + while True: + try: + wire, _ = z_socket.recvfrom(65535) + except TimeoutError: + break + received.append(decode_rudp_packet(wire).packet_type) + self.assertEqual(received, [CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_packet_the_session_refuses_ends_only_that_peers_session(self) -> None: + from cultnet_py import CultNetRudpPacket, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + # A fragment whose metadata the session refuses: fragment id 0 names no set. + refused = CultNetRudpPacket( + packet_type=CultNetRudpPacketType.DATA, + connection_id=server.rudp_connection_id, + sequence=0, + ack=0, + ack_mask=0, + channel_id="schema", + reliable=False, + fragment_id=0, + fragment_index=0, + fragment_count=2, + payload=b"x", + ) + x.socket.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + finally: + server.stop() + + def test_a_message_whose_handling_fails_ends_only_that_peers_session(self) -> None: + import msgpack # type: ignore + + canary = "canary-7f3e1c" + # Well-formed frames whose fields have the wrong type: handling raises TypeError. + frames = [ + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "schemaIds": 5}, + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "recordKeys": True}, + ] + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + y, _ = self._client(server, "peer-y") + for frame in frames: + with self.subTest(frame=frame): + x, _ = self._client(server, "peer-x") + with self.assertLogs("cultmesh_py.server", level="WARNING") as logs: + x.send("schema", msgpack.packb(frame, use_bin_type=True)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + self.assertEqual(len(logs.output), 1) + self.assertIn("TypeError", logs.output[0]) + self.assertNotIn(canary, logs.output[0]) + finally: + server.stop() + + def _raw_peer(self, server: Any) -> tuple[socket.socket, Any]: + """A peer driven packet by packet, to deliver frames in an order a client would not.""" + from cultnet_py import CultNetRudpSession, CultNetRudpSessionOptions, encode_rudp_packet + + sock = bind_udp_socket() + sock.settimeout(0.05) + session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=server.rudp_connection_id)) + sock.sendto(encode_rudp_packet(session.create_connect(0, b"join")), ("127.0.0.1", server.port)) + wire, _ = sock.recvfrom(65535) + accept = decode_rudp_packet(wire) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + session.receive(accept, 0) + return sock, session + + def test_the_goodbye_for_a_refused_reliable_frame_does_not_acknowledge_it(self) -> None: + import dataclasses + + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + sock, session = self._raw_peer(server) + # A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + # The session records its sequence before it refuses it. + (fragment,) = session.send_many("schema", b"x", CultNetRudpSendOptions(reliable=True, ordered=True)) + refused = dataclasses.replace(fragment, fragment_id=0, fragment_index=0, fragment_count=2) + sock.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + goodbye = None + for _ in range(40): + try: + wire, _ = sock.recvfrom(65535) + except TimeoutError: + continue + packet = decode_rudp_packet(wire) + if packet.packet_type == CultNetRudpPacketType.DISCONNECT: + goodbye = packet + break + self.assertIsNotNone(goodbye) + self.assertEqual(goodbye.payload, b"session refused a packet") + distance = goodbye.ack - refused.sequence + acknowledged = distance == 0 or (0 < distance <= 32 and goodbye.ack_mask & (1 << (distance - 1)) != 0) + self.assertFalse(acknowledged, "the goodbye acknowledged the refused frame") + finally: + server.stop() + + def test_responses_after_one_that_can_never_be_sent_are_not_sent(self) -> None: + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + sock, session = self._raw_peer(server) + addr = sock.getsockname()[:2] + request = hello(runtime_id="probe").to_bytes() + options = CultNetRudpSendOptions(reliable=True, ordered=True) + first = session.send("schema", request, options) + second = session.send("schema", request, options) + attempts: list[CultNetRudpPacketType] = [] + failures = [1] + + def sendto(sock_: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(addr): + packet_type = decode_rudp_packet(data).packet_type + attempts.append(packet_type) + if packet_type == CultNetRudpPacketType.DATA and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock_, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The second frame waits for the first, so one receive delivers both, and + # the first of the two responses can never be sent as built. + sock.sendto(encode_rudp_packet(second), ("127.0.0.1", server.port)) + time.sleep(0.1) + attempts.clear() + sock.sendto(encode_rudp_packet(first), ("127.0.0.1", server.port)) + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.1) + + # The unsendable response, then only the goodbye: no second response, no ack. + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + mode = ["transient"] + attempts: list[CultNetRudpPacketType] = [] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + packet_type = decode_rudp_packet(data).packet_type + if mode[0] != "transient": + attempts.append(packet_type) + # Only responses fail; X's requests are acknowledged, so X goes quiet. + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + x.send("schema", hello(runtime_id="one").to_bytes()) + x.send("schema", hello(runtime_id="two").to_bytes()) + for _ in range(40): + x.receive_once() + time.sleep(0.005) + self._wait_for(lambda: server.rudp_send_failures >= 2) + # Both responses are owed; the next resend of the first can never be sent. + mode[0] = "permanent" + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.2) + + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def _wait_for(self, predicate: Callable[[], bool]) -> None: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.005) + self.fail("condition not reached") + + if __name__ == "__main__": unittest.main() diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 8a2706fd..6c2bf639 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,9 +3,11 @@ use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, + encode_rudp_packet, is_permanent_send_error, }; use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -234,6 +236,10 @@ pub enum CultMeshRudpRejectionReason { PayloadBudgetFull, /// The session could not queue the response. Carries the session's error text. ResponseQueueFailed(String), + /// A packet of the response can never be sent to the peer as built + /// (`is_permanent_send_error`). The session ends with a goodbye naming the + /// error, and no refusal: the server knows the peer cannot be reached. + ResponseSendFailed(std::io::ErrorKind), } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -270,6 +276,9 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::ResponseQueueFailed(error) => { write!(f, "snapshot response could not be queued: {error}") } + Self::ResponseSendFailed(kind) => { + write!(f, "snapshot response could not be sent: {kind}") + } } } } @@ -277,7 +286,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 8] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 9] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -286,6 +295,7 @@ pub(crate) const REFUSAL_TEXTS: [&str; 8] = [ "the snapshot response could not be encoded", "the retained payload budget is full", "the snapshot response could not be queued", + "the snapshot response could not be sent", ]; impl CultMeshRudpRejectionReason { @@ -300,6 +310,7 @@ impl CultMeshRudpRejectionReason { Self::ResponseEncodingFailed(_) => 5, Self::PayloadBudgetFull => 6, Self::ResponseQueueFailed(_) => 7, + Self::ResponseSendFailed(_) => 8, }] } } @@ -319,6 +330,16 @@ pub enum CultMeshRudpPollOutcome { ApplicationRejected(CultMeshRudpApplicationRejection), } +/// Why a session ends before its peer is done with it. +enum SessionEnd { + /// The session refused a packet. + Refused, + /// The server rejected a peer's message; the peer is sent this fixed text. + Rejected(&'static str), + /// The session owes its peer a packet that can never be sent as built. + Unsendable(std::io::Error), +} + struct SessionEntry { session: CultNetRudpSession, created_at_monotonic_millis: u64, @@ -338,6 +359,14 @@ pub struct CultMeshRudpDocumentServer { clock: C, options: CultMeshRudpDocumentServerOptions, packets_dropped: u64, + send_failures: u64, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultMeshRudpDocumentServer @@ -366,6 +395,11 @@ where clock, options, packets_dropped: 0, + send_failures: 0, + #[cfg(test)] + failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -377,6 +411,18 @@ where self.packets_dropped } + /// Datagrams that could not be sent to a peer and may yet be: no route, full + /// buffers, a refused path. Each is that peer's lost datagram: a reliable + /// packet stays pending and is resent, and the peer's session ends by idle + /// timeout, lifetime, refusal or Disconnect, never by the failure. Never an + /// error: one unreachable peer must not stop the poll that serves the + /// others. A datagram that can never be sent as built + /// (`is_permanent_send_error`) is not counted here: it ends that peer's + /// session, and the peer's goodbye names the error. + pub fn send_failures(&self) -> u64 { + self.send_failures + } + pub fn local_addr(&self) -> Result { Ok(self.socket.local_addr()?) } @@ -399,7 +445,14 @@ where let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { Ok(value) => value, - Err(error) if error.kind() == ErrorKind::WouldBlock => { + // Windows reports an earlier ICMP port-unreachable on the next + // receive; it names no datagram of this poll. + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::ConnectionReset + ) => + { return Ok(CultMeshRudpPollOutcome::Idle); } Err(error) => return Err(error.into()), @@ -459,15 +512,16 @@ where let result = match result { Ok(result) => result, Err(_) => { - self.end_refused_session(key)?; + self.end_session(key, SessionEnd::Refused)?; self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } }; - if let Some(reply) = result.reply { - self.send_packet(key.remote_addr, &reply)?; - } + let mut unsendable = match result.reply { + Some(reply) => self.send_packet(key.remote_addr, &reply)?, + None => None, + }; if result.disconnected { self.sessions.remove(&key); @@ -492,19 +546,28 @@ where now_unix, now_monotonic, )? { - self.end_rejected_session(key, rejection.reason.refusal_text())?; + let end = match rejection.reason { + CultMeshRudpRejectionReason::ResponseSendFailed(kind) => { + SessionEnd::Unsendable(std::io::Error::from(kind)) + } + ref reason => SessionEnd::Rejected(reason.refusal_text()), + }; + self.end_session(key, end)?; return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } } - if packet.reliable { + if packet.reliable && unsendable.is_none() { let ack = self .sessions .get_mut(&key) .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before ACK"))? .session .create_ack(); - self.send_packet(key.remote_addr, &ack)?; + unsendable = self.send_packet(key.remote_addr, &ack)?; + } + if let Some(error) = unsendable { + self.end_session(key, SessionEnd::Unsendable(error))?; } Ok(CultMeshRudpPollOutcome::Handled) } @@ -524,11 +587,17 @@ where let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { for packet in entry.session.due_resends(now) { - resends.push((key.remote_addr, packet)); + resends.push((*key, packet)); } } - for (remote_addr, packet) in &resends { - self.send_packet(*remote_addr, packet)?; + for (key, packet) in &resends { + // A session ended earlier in this loop has nothing left to resend. + if !self.sessions.contains_key(key) { + continue; + } + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + self.end_session(*key, SessionEnd::Unsendable(error))?; + } } Ok(CultMeshRudpMaintenance { sessions_expired: expired, @@ -560,7 +629,7 @@ where match entry.session.accept_connect(packet, now, Vec::new()) { Ok(reply) => reply, Err(_) => { - self.end_refused_session(key)?; + self.end_session(key, SessionEnd::Refused)?; return Ok(false); } } @@ -590,7 +659,10 @@ where accept } }; - self.send_packet(key.remote_addr, &reply)?; + if let Some(error) = self.send_packet(key.remote_addr, &reply)? { + self.end_session(key, SessionEnd::Unsendable(error))?; + return Ok(false); + } Ok(true) } @@ -765,7 +837,14 @@ where entry.admitted_payload_bytes = entry.admitted_payload_bytes.saturating_add(payload_bytes); for packet in &packets { - self.send_packet(key.remote_addr, packet)?; + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: CultMeshRudpRejectionReason::ResponseSendFailed(error.kind()), + })); + } } } _ => {} @@ -773,36 +852,56 @@ where Ok(None) } - /// The session refused a packet: it ends, and the client is told so it does - /// not wait on a session the server no longer holds. - fn end_refused_session(&mut self, key: CultMeshRudpSessionKey) -> Result<()> { + /// Ends a session before its peer is done with it, and says goodbye. It is + /// the one way a session ends early: every end removes the session, sends + /// what the peer is owed, then the goodbye. + /// + /// A rejected message's refusal goes first, unreliable and unordered: the + /// session ends with it, so it is never resent, and a gap in what the peer + /// has received must not hold it back. It is built before the reset and + /// carries acknowledgement fields built after it, which acknowledge + /// nothing: a publisher reads an acknowledgement as admission. A refusal + /// that can never be sent as built is not dropped: the goodbye names its + /// error instead. A refusal that cannot be encoded is not sent. + fn end_session(&mut self, key: CultMeshRudpSessionKey, end: SessionEnd) -> Result<()> { let Some(mut entry) = self.sessions.remove(&key) else { return Ok(()); }; - let goodbye = entry.session.end_refused_session(); - self.send_packet(key.remote_addr, &goodbye) + let (refusal, mut unsendable) = match end { + SessionEnd::Refused => (Vec::new(), None), + SessionEnd::Rejected(text) => (self.refusal_packets(&mut entry.session, text), None), + SessionEnd::Unsendable(error) => (Vec::new(), Some(error)), + }; + entry.session.reset_peer_state(); + let nothing = entry.session.create_ack(); + for mut packet in refusal { + packet.ack = nothing.ack; + packet.ack_mask = nothing.ack_mask; + if let Some(error) = self.send_packet(key.remote_addr, &packet)? { + unsendable = Some(error); + break; + } + } + let goodbye = match &unsendable { + Some(error) => entry.session.end_unsendable_session(error), + None => entry.session.end_refused_session(), + }; + // The goodbye is the session's last datagram. If it too can never be + // sent, nobody is left to tell. + self.send_packet(key.remote_addr, &goodbye)?; + Ok(()) } - /// The server refused a peer's message: the session ends, and the peer is - /// sent the refusal before the goodbye, so it can tell a refusal from loss. - /// The refusal is unreliable and unordered: the session ends with it, so it - /// is never resent, and a gap in what the peer has received must not hold - /// it back. It carries the goodbye's acknowledgement fields, which the - /// reset has emptied: it must not acknowledge the refused message, because - /// a publisher reads that acknowledgement as admission. A refusal that - /// cannot be encoded is not sent; the goodbye still is. - fn end_rejected_session(&mut self, key: CultMeshRudpSessionKey, reason: &str) -> Result<()> { - let Some(mut entry) = self.sessions.remove(&key) else { - return Ok(()); - }; + /// The refusal a rejected peer is sent, as unreliable, unordered packets. + fn refusal_packets(&self, session: &mut CultNetRudpSession, text: &str) -> Vec { let refusal = CultNetMessage::Error { - error: reason.into(), + error: text.into(), code: None, details: None, }; - let packets = encode_cultnet_message_to_vec(&refusal, CultNetWireContract::CultNetSchemaV0) + encode_cultnet_message_to_vec(&refusal, CultNetWireContract::CultNetSchemaV0) .and_then(|payload| { - entry.session.send_many( + session.send_many( "schema", payload, CultNetRudpSendOptions { @@ -815,14 +914,7 @@ where Some(self.options.max_fragment_bytes), ) }) - .unwrap_or_default(); - let goodbye = entry.session.end_refused_session(); - for mut packet in packets { - packet.ack = goodbye.ack; - packet.ack_mask = goodbye.ack_mask; - self.send_packet(key.remote_addr, &packet)?; - } - self.send_packet(key.remote_addr, &goodbye) + .unwrap_or_default() } /// The fragments `send_many` splits a response of `bytes` into. @@ -859,13 +951,48 @@ where .is_some_and(|total| total <= self.options.max_admitted_payload_bytes) } - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A permanent one (`is_permanent_send_error`) is returned as `Some` + /// for the caller to end the session over. Only an encode failure, the + /// server's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; - self.socket.send_to(&wire, remote_addr)?; - Ok(()) + match self.send_datagram(&wire, remote_addr) { + Ok(_) => {} + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), + Err(_) => self.send_failures += 1, + } + Ok(None) + } + + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + { + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + } + self.socket.send_to(wire, remote_addr) } } +#[cfg(test)] +mod send_failure_tests; + fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.max_sessions == 0 { return Err(anyhow!("max_sessions must be greater than zero")); diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs new file mode 100644 index 00000000..8c48a092 --- /dev/null +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -0,0 +1,458 @@ +//! One peer whose datagrams cannot be sent must not stop the server serving +//! the others. The failure is injected in `send_packet` through +//! `failing_peers`; a loopback peer cannot be made unroutable on demand. + +use super::*; +use cultnet_rs::{ + CultNetRawPayloadEncoding, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, +}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; + +#[derive(Clone)] +struct Clock(Arc); + +impl CultMeshRudpServerClock for Clock { + fn now_unix_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + fn now_monotonic_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } +} + +type Received = Arc>>; +type Server = CultMeshRudpDocumentServer< + Box Result<()>>, + Box Result>>, + Clock, +>; +type Client = CultNetRudpSocketTransportConnection; + +fn document(key: &str) -> CultNetRawDocumentRecord { + CultNetRawDocumentRecord { + schema_id: "test.raw.v1".into(), + record_key: key.into(), + stored_at: "2026-09-03T00:00:00Z".into(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: vec![0x81, 0xa1, 0x78, 0x2a], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } +} + +fn client(target: SocketAddr, id: u32) -> (Client, SocketAddr) { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let addr = socket.local_addr().unwrap(); + let client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "test-client".into(), + socket, + mode: CultNetRudpSocketMode::Client, + remote_addr: Some(target), + connection_id: id, + initial_sequence: Some(1), + resend_delay_ms: 10, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(1200), + max_pending_reliable_packets: Some(64), + media_reliable_expire_after_ms: None, + reconnect_policy: None, + }) + .unwrap(); + (client, addr) +} + +fn send(client: &mut Client, message: &CultNetMessage) { + client + .send( + "schema", + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap(), + ) + .unwrap(); +} + +/// A server with a 1 s idle timeout and two connected peers, the first +/// returned being the one whose address sorts first. +fn two_peers() -> ( + Server, + Clock, + Received, + (Client, SocketAddr), + (Client, SocketAddr), +) { + let clock = Clock(Arc::new(AtomicU64::new(1_000))); + let received: Received = Arc::default(); + let sink_received = received.clone(); + let sink: Box Result<()>> = + Box::new(move |receipt| { + sink_received.lock().unwrap().push(receipt.message_id); + Ok(()) + }); + let source: Box< + dyn FnMut(&CultMeshRudpSnapshotQuery) -> Result>, + > = Box::new(|_| Ok(vec![document("catalog")])); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0").unwrap(), + sink, + source, + clock.clone(), + CultMeshRudpDocumentServerOptions { + session_idle_timeout: Duration::from_secs(1), + resend_delay: Duration::from_millis(10), + ..Default::default() + }, + ) + .unwrap(); + let target = server.local_addr().unwrap(); + let (mut a, a_addr) = client(target, 7); + let (mut b, b_addr) = client(target, 7); + a.connect(Vec::new()).unwrap(); + b.connect(Vec::new()).unwrap(); + for _ in 0..500 { + server.poll_once().unwrap(); + a.receive_once().unwrap(); + b.receive_once().unwrap(); + if a.connected() && b.connected() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert!(a.connected() && b.connected()); + for _ in 0..4 { + server.poll_once().unwrap(); + } + assert_eq!(server.session_count(), 2); + if a_addr < b_addr { + (server, clock, received, (a, a_addr), (b, b_addr)) + } else { + (server, clock, received, (b, b_addr), (a, a_addr)) + } +} + +#[test] +fn one_peer_whose_sends_fail_does_not_stop_the_server_serving_the_others() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + + // X's reply, ack and snapshot response all fail to send. X is visited first. + send( + &mut x, + &CultNetMessage::SnapshotRequest { + message_id: "x-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put".into(), + document: document("y"), + }, + ); + send( + &mut y, + &CultNetMessage::SnapshotRequest { + message_id: "y-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + let mut y_response = None; + for step in 0..200 { + // The clock moves past X's resend delay so `maintain` has X's resend due. + clock.0.store(1_000 + step * 15, Ordering::SeqCst); + // Never `Err`: X's failed sends are its own lost datagrams. + server.poll_once().unwrap(); + if let Some(frame) = y.receive_once().unwrap() { + y_response = Some(frame); + break; + } + thread::sleep(Duration::from_millis(1)); + } + let frame = y_response.expect("Y receives its snapshot response"); + let response = + decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + assert!(matches!( + response, + CultNetMessage::SnapshotResponseRaw { ref message_id, .. } if message_id == "y-snapshot" + )); + assert!(server.send_failures() > 0); + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); + assert!(server.maintain().is_ok()); + + // X's session ends by the idle rule that already ends sessions, not by the + // failure: Y stays active, X does not. + clock.0.store(1_900, Ordering::SeqCst); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put-2".into(), + document: document("y2"), + }, + ); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2, "the failure ended no session"); + clock.0.store(2_500, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + assert_eq!(maintenance.sessions_expired, 1); + assert_eq!(server.session_count(), 1); +} + +// A permanent send failure is one that can never succeed for the datagram as +// built (`is_permanent_send_error`). It ends that peer's session, never the +// poll, and the peer's goodbye names the error. + +fn put(id: &str) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document(id), + } +} + +fn snapshot(id: &str) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + } +} + +fn goodbye_reason(client: &mut Client) -> Vec { + for _ in 0..50 { + client.receive_once().unwrap(); + if let Some(reason) = client.disconnect_reason() { + return reason.to_vec(); + } + thread::sleep(Duration::from_millis(2)); + } + panic!("the client was never told the session ended"); +} + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +#[test] +fn a_permanent_failure_acknowledging_a_document_ends_the_session_after_the_document_is_kept() { + let (mut server, _clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &put("x-put")); + send(&mut y, &put("y-put")); + + for _ in 0..10 { + // Never `Err`: the failure ends X's session, not the poll. + server.poll_once().unwrap(); + } + let mut kept = received.lock().unwrap().clone(); + kept.sort(); + assert_eq!(kept, ["x-put", "y-put"]); + assert_eq!(server.session_count(), 1); + assert_eq!(server.send_failures(), 0); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); +} + +#[test] +fn a_permanent_failure_sending_a_snapshot_response_rejects_it_and_sends_no_refusal() { + let (mut server, _clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &snapshot("x-snapshot")); + + let mut rejection = None; + for _ in 0..10 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once().unwrap() { + rejection = Some(found); + break; + } + } + let rejection = rejection.expect("the response can never be sent"); + assert_eq!(rejection.operation, CultMeshRudpApplicationOperation::SnapshotRequest); + assert_eq!(rejection.message_id, "x-snapshot"); + assert_eq!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(ErrorKind::InvalidInput) + ); + assert_eq!(server.session_count(), 1); + // The server knows X cannot be reached: X is sent no refusal, only the + // goodbye naming the error, which carries no text of the peer's. + let mut frames = Vec::new(); + let mut reason = None; + for _ in 0..50 { + if let Some(frame) = x.receive_once().unwrap() { + frames.push(frame); + } + if let Some(found) = x.disconnect_reason() { + reason = Some(found.to_vec()); + break; + } + thread::sleep(Duration::from_millis(2)); + } + assert!(frames.is_empty(), "no refusal is sent to an unreachable peer: {frames:?}"); + assert!(is_unsendable_reason(&reason.expect("the client was told the session ended"))); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + // X's snapshot response is lost, so a resend is due; then the datagram can + // never be sent as built. + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + server.failing_peers.clear(); + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + + let maintenance = server.maintain().unwrap(); + assert!(maintenance.packets_resent > 0); + assert_eq!(server.session_count(), 1); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); + + send(&mut y, &put("y-put")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let (mut z, z_addr) = client(server.local_addr().unwrap(), 9); + server.unsendable_after.insert(z_addr, 0); + z.connect(Vec::new()).unwrap(); + + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2); + assert_eq!(server.packets_dropped(), 1); +} + +#[test] +fn healthy_sends_are_not_failures() { + let (mut server, _clock, _received, (mut x, _), (_y, _)) = two_peers(); + send(&mut x, &put("x-put")); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.send_failures(), 0); + assert_eq!(server.session_count(), 2); +} + +#[test] +fn a_session_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut server, clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("one")); + send(&mut x, &snapshot("two")); + for _ in 0..10 { + server.poll_once().unwrap(); + } + let before = server.send_failures(); + + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + + // Both responses are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(maintenance.packets_resent, 2); + assert_eq!(server.send_failures() - before, 1); + assert_eq!(server.session_count(), 1); +} + +#[test] +fn no_ack_follows_a_reply_that_can_never_be_sent() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let target = server.local_addr().unwrap(); + // A peer driven packet by packet: a client never sends a reliable Ping, but a + // peer may, and the server both answers it and acknowledges it. + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let addr = socket.local_addr().unwrap(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + ..Default::default() + }); + let connect = session.create_connect(0, Vec::new()).unwrap(); + socket + .send_to(&encode_rudp_packet(&connect).unwrap(), target) + .unwrap(); + let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; + for _ in 0..10 { + server.poll_once().unwrap(); + } + let (read, _) = socket.recv_from(&mut wire).unwrap(); + let accept = decode_rudp_packet(&wire[..read]).unwrap(); + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + session.receive(&accept, 0).unwrap(); + assert_eq!(server.session_count(), 3); + + let mut ping = session.create_ping(b"p".to_vec()); + ping.reliable = true; + ping.sequence = connect.sequence.wrapping_add(1); + // The Pong can never be sent; the ack after it could. + server.unsendable_after.insert(addr, 0); + socket + .send_to(&encode_rudp_packet(&ping).unwrap(), target) + .unwrap(); + for _ in 0..10 { + server.poll_once().unwrap(); + } + + // The session ends over the Pong: the peer gets its goodbye and nothing else. + assert_eq!(server.session_count(), 2); + let mut arrived = Vec::new(); + while let Ok((read, _)) = socket.recv_from(&mut wire) { + arrived.push(decode_rudp_packet(&wire[..read]).unwrap()); + } + assert_eq!(arrived.len(), 1, "{arrived:?}"); + assert_eq!(arrived[0].packet_type, CultNetRudpPacketType::Disconnect); + assert!(is_unsendable_reason(&arrived[0].payload)); +} + +/// Windows reports an ICMP port-unreachable for an earlier datagram as +/// `ConnectionReset` on the next receive. It names no datagram of this poll, so +/// the poll is idle and the server goes on serving. Only Windows reports it, so +/// only a Windows run can see the rule break. +#[cfg(windows)] +#[test] +fn a_reset_reported_by_a_receive_is_idle_not_an_error() { + let (mut server, _clock, received, (mut x, _), (mut y, _)) = two_peers(); + // X asks for a snapshot and goes away before the answer. + send(&mut x, &snapshot("x-snapshot")); + drop(x); + for _ in 0..20 { + server + .poll_once() + .expect("a reset from X's closed port is not an error"); + thread::sleep(Duration::from_millis(1)); + } + + send(&mut y, &put("y-put")); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index a3f5abcc..596c08ba 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -1492,3 +1492,53 @@ fn a_refusal_reaches_a_peer_missing_an_earlier_packet() -> Result<()> { assert!(disconnected, "the refusal is followed by a goodbye"); Ok(()) } + +/// A snapshot the reliable queue cannot take is answered with a refusal, not +/// left to time out: the refusal needs no room in that queue. +#[test] +fn a_snapshot_the_queue_cannot_take_is_answered() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_fragment_bytes: 100, + max_pending_reliable_packets_per_session: 8, + max_snapshot_response_bytes: 8192, + ..Default::default() + }; + // Each document alone is served in 800 bytes, eight fragments; both need sixteen. + let source = Source::documents(vec![ + document_served_at("q1", 800)?, + document_served_at("q2", 800)?, + ]); + let mut server = server(options, Clock::new(72_000), Sink::default(), source)?; + let mut client = client(server.local_addr()?, 151)?; + connect(&mut server, &mut [&mut client])?; + let receipt = send_reliable( + &mut client, + &CultNetMessage::SnapshotRequest { + message_id: "both".into(), + schema_ids: None, + record_keys: None, + }, + )?; + let mut rejection = None; + for _ in 0..100 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once()? { + rejection = Some(found); + break; + } + client.receive_once()?; + } + let rejection = rejection.expect("the response cannot be queued"); + assert!( + matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseQueueFailed(_) + ), + "{:?}", + rejection.reason + ); + assert_eq!( + refusal_seen_by(&mut client, &receipt)?, + "the snapshot response could not be queued" + ); + Ok(()) +} diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 689fbed2..245e8aa6 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -14,6 +14,7 @@ import { CultNetRudpSession, CultNetPeer, CultNetRudpSocketTransportConnection, + rudpClientBindHost, CultNetSchemaCatalog, CultNetShardCatalog, cultNetBuiltinSchemaRegistry, @@ -21,7 +22,9 @@ import { defineCultNetDocumentBinding, encodeCultNetMessageForWire, encodeRudpPacket, + isPermanentSendError, parseCultNetMessage, + sendRudpDatagram, type CultNetErrorMessage, type CultNetDocumentBinding, type CultNetMessage, @@ -4303,7 +4306,7 @@ export interface CultMeshRudpDocumentReceipt; close(): void; } @@ -4388,6 +4397,11 @@ export interface CultMeshRudpEndpoint { } export interface CultMeshRudpSocketOptions { + /** + * Local address to bind. Unset, a server binds loopback, and a client binds + * loopback for a loopback endpoint and the unspecified address of the + * endpoint's family otherwise, so it can reach a remote host. + */ bindHost?: string; bindPort?: number; socket?: Socket; @@ -5819,15 +5833,19 @@ export class CultMesh { const port = options.bindPort ?? 0; const bind = { host, port }; const socket = options.socket ?? createSocket(host.includes(":") ? "udp6" : "udp4"); - const sessions = new Map; - }>(); + }; + const sessions = new Map(); let sessionSequence = 0; + let sendFailures = 0; const resendPollMs = Math.max(10, options.resendPollMs ?? 25); const sessionTimeoutMs = Math.max(1_000, options.sessionTimeoutMs ?? 30_000); const wireContract = options.wireContract ?? "cultnet.schema.v0"; @@ -5863,15 +5881,15 @@ export class CultMesh { if (packet.packetType === "connect") { const connectPayload = Uint8Array.from(packet.payload ?? []); if (record?.session.connectRepeats(packet)) { - sendPacket(record.session.acceptConnect(packet, nowMs), remote); + sendPacket(record, record.session.acceptConnect(packet, nowMs)); return; } - if (record) closeSession(record); - record = { + const next: SessionRecord = { sessionId: `${runtimeId}:${++sessionSequence}`, remote: { address: remote.address, family: remote.family, port: remote.port }, connectPayload, closed: false, + unsendable: false, session: new CultNetRudpSession({ connectionId, initialSequence: options.initialSequence, @@ -5880,8 +5898,12 @@ export class CultMesh { }), work: Promise.resolve(), }; - sessions.set(key, record); - sendPacket(record.session.acceptConnect(packet, nowMs), remote); + sendPacket(next, next.session.acceptConnect(packet, nowMs)); + // A peer whose Accept can never be sent cannot be answered: no session + // starts, and one already there is left as it was. + if (next.unsendable) return; + if (record) closeSession(record); + sessions.set(key, next); return; } @@ -5889,12 +5911,22 @@ export class CultMesh { return; } - const result = record.session.receive(packet, nowMs); + let result: ReturnType; + try { + result = record.session.receive(packet, nowMs); + } catch { + // receive() has already recorded the packet's reliable sequence, so the + // session cannot be kept: a retransmit would be acknowledged and the frame + // silently lost. It ends, never the server, and the peer is told. What a + // datagram carries is the sender's business, so nothing is reported. + endRefusedSession(key, record); + return; + } if (result.reply) { - sendPacket(result.reply, record.remote); + sendPacket(record, result.reply); } for (const ready of result.readyToSend ?? []) { - sendPacket(ready, record.remote); + sendPacket(record, ready); } for (const frame of result.delivered) { if (frame.channelId !== "schema") { @@ -5910,7 +5942,7 @@ export class CultMesh { return; } if (packet.reliable) { - sendPacket(record.session.createAckForReceived(packet.sequence), record.remote); + sendPacket(record, record.session.createAckForReceived(packet.sequence)); } } catch (error) { reportError(error); @@ -5919,7 +5951,7 @@ export class CultMesh { socket.on("error", reportError); async function handleDocumentServerFrame( - record: { sessionId: string; remote: { address: string; family: string; port: number }; session: CultNetRudpSession; connectPayload: Uint8Array; closed: boolean }, + record: SessionRecord, payload: Uint8Array, ): Promise { if (record.closed) return; @@ -5938,10 +5970,12 @@ export class CultMesh { sendSchemaMessage(record, options.documents.createRawSnapshotResponse(cache, message.messageId, message)); } catch (error) { reportError(error); + // Unreliable and unordered: when the reply could not be queued, the answer must not + // need room in the queue that just refused it. sendSchemaMessage(record, { schemaVersion: "cultnet.error.v0", error: error instanceof Error ? error.message : String(error), - }); + }, { reliable: false }); } return; } @@ -5989,13 +6023,7 @@ export class CultMesh { } } - function publicSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): CultMeshRudpServerSession { + function publicSession(record: SessionRecord): CultMeshRudpServerSession { return { sessionId: record.sessionId, remote: { ...record.remote }, @@ -6007,32 +6035,60 @@ export class CultMesh { }; } - function notifySessionClosed(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function notifySessionClosed(record: SessionRecord): void { if (!options.onSessionClosed) return; void Promise.resolve(options.onSessionClosed(publicSession(record))).catch(reportError); } - function closeSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function closeSession(record: SessionRecord): void { if (record.closed) return; record.closed = true; notifySessionClosed(record); } + /** + * Sends one packet to a session's peer. A failure is that peer's alone: one that may + * pass (no route, full buffers) is a lost datagram, counted, and resent if it was + * reliable; one that can never pass as built ends that peer's session. Neither ends + * the server, a timer or another peer's session. + */ + function sendPacket(record: SessionRecord, packet: CultNetRudpPacket): void { + sendRudpDatagram(socket, encodeRudpPacket(packet), record.remote.port, record.remote.address, (error) => { + if (isPermanentSendError(error)) { + endUnsendableSession(record, error); + } else { + sendFailures += 1; + } + }); + } + + function endRefusedSession(key: string, record: SessionRecord): void { + sessions.delete(key); + closeSession(record); + // The goodbye is built after the reset, or its ack field would acknowledge the + // very frame the session refused. + record.session.resetPeerState(); + const goodbye = encodeRudpPacket(record.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + + /** The session owes its peer a packet that can never be sent as built. */ + function endUnsendableSession(record: SessionRecord, error: Error): void { + if (record.unsendable) return; + record.unsendable = true; + const key = `${record.remote.address}:${record.remote.port}`; + // A session that never started, or was already replaced, has nothing to end. + if (sessions.get(key) !== record) return; + sessions.delete(key); + closeSession(record); + const goodbye = encodeRudpPacket(record.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + function sendSchemaMessage( - record: { sessionId: string; remote: { address: string; port: number }; session: CultNetRudpSession; closed: boolean }, + record: SessionRecord, message: CultNetMessage, + { reliable = true }: { reliable?: boolean } = {}, ): void { if (record.closed) throw new Error(`CultMesh RUDP session ${record.sessionId} is closed.`); const payload = encodeSchemaMessage(message); @@ -6043,12 +6099,12 @@ export class CultMesh { ); } for (const packet of record.session.sendMany("schema", payload, { - reliable: true, - ordered: true, + reliable, + ordered: reliable, nowMs: Date.now(), maxFragmentBytes, })) { - sendPacket(packet, record.remote); + sendPacket(record, packet); } } @@ -6057,13 +6113,6 @@ export class CultMesh { return encode(encodeCultNetMessageForWire(message, wireContract)); } - /** A datagram that fails to leave reaches onError; nothing vanishes. */ - function sendPacket(packet: CultNetRudpPacket, remote: { address: string; port: number }): void { - socket.send(encodeRudpPacket(packet), remote.port, remote.address, (error) => { - if (error) reportError(error); - }); - } - /** Builds the receipt `onDocumentPutRaw` returned, under the put's message id by default. */ function receiptMessage(receipt: CultMeshRudpDocumentReceipt, putMessageId: string): CultNetMessage { return options.documents.createRawDocumentPutMessage( @@ -6115,6 +6164,9 @@ export class CultMesh { return { bind, + get sendFailures() { + return sendFailures; + }, async start(): Promise { await new Promise((resolve, reject) => { socket.once("error", reject); @@ -6136,7 +6188,7 @@ export class CultMesh { continue; } for (const packet of record.session.dueResends(nowMs)) { - sendPacket(packet, record.remote); + sendPacket(record, packet); } } }, resendPollMs); @@ -6272,7 +6324,7 @@ export class CultMesh { options: CultMeshRudpSocketOptions = {}, ): Promise { requireNonEmpty(runtimeId, "runtimeId"); - const socket = options.socket ?? (await bindRudpSocket(options)); + const socket = options.socket ?? (await bindRudpSocket(options.bindHost ?? "127.0.0.1", options)); return new CultNetRudpSocketTransportConnection({ runtimeId, socket, @@ -6298,7 +6350,10 @@ export class CultMesh { requireNonEmpty(runtimeId, "runtimeId"); const parsedEndpoint = typeof endpoint === "string" ? CultMesh.parseRudpEndpoint(endpoint) : endpoint; - const socket = options.socket ?? (await bindRudpSocket(options)); + const socket = options.socket ?? (await bindRudpSocket( + options.bindHost ?? (await rudpClientBindHost(parsedEndpoint.host)), + options, + )); return new CultNetRudpSocketTransportConnection({ runtimeId, socket, @@ -6723,9 +6778,8 @@ function toUint8Array(value: unknown): Uint8Array { throw new Error("CultNet raw document payload was not binary."); } -async function bindRudpSocket(options: CultMeshRudpSocketOptions): Promise { - const socket = createSocket("udp4"); - const host = options.bindHost ?? "127.0.0.1"; +async function bindRudpSocket(host: string, options: CultMeshRudpSocketOptions): Promise { + const socket = createSocket(host.includes(":") ? "udp6" : "udp4"); const port = options.bindPort ?? 0; await new Promise((resolve, reject) => { socket.once("error", reject); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 9281355c..7f4ee959 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -2550,6 +2550,44 @@ test("CultMesh TS branded facade exposes schema and shard catalogs", () => { ); }); +async function rudpClientBoundHost(endpoint: string, bindHost?: string): Promise { + const client = await CultMesh.createRudpClient("cultmesh-ts-bind-client", 0x10203060, endpoint, { bindHost }); + try { + return client.profile.transports[0]?.host; + } finally { + client.close(); + } +} + +// A client bound to loopback cannot send to another host on Windows, so a +// client for a remote endpoint binds the unspecified address of its family. +test("CultMesh TS RUDP client for a remote endpoint binds the unspecified address", async () => { + assert.equal(await rudpClientBoundHost("rudp://10.77.0.1:17872"), "0.0.0.0"); + assert.equal(await rudpClientBoundHost("rudp://[2001:db8::1]:17872"), "::"); +}); + +test("CultMesh TS RUDP client for a loopback endpoint binds loopback", async () => { + assert.equal(await rudpClientBoundHost("rudp://127.0.0.1:17872"), "127.0.0.1"); + assert.equal(await rudpClientBoundHost("rudp://[::1]:17872"), "::1"); +}); + +test("CultMesh TS RUDP client binds an explicit bindHost whatever the endpoint", async () => { + assert.equal(await rudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1"), "127.0.0.1"); + assert.equal(await rudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0"), "0.0.0.0"); +}); + +test("CultMesh TS RUDP server binds loopback unless told otherwise", async () => { + const loopback = await CultMesh.createRudpServer("cultmesh-ts-bind-server", 0x10203061); + const any = await CultMesh.createRudpServer("cultmesh-ts-bind-server", 0x10203062, { bindHost: "0.0.0.0" }); + try { + assert.equal(loopback.profile.transports[0]?.host, "127.0.0.1"); + assert.equal(any.profile.transports[0]?.host, "0.0.0.0"); + } finally { + loopback.close(); + any.close(); + } +}); + test("CultMesh TS branded facade creates RUDP clients from peer endpoints", async () => { const connectionId = 0x10203044; const server = await CultMesh.createRudpServer( @@ -3456,6 +3494,24 @@ test("CultMesh TS RUDP document server refuses a put whose reply would overflow }); }); +test("CultMesh TS RUDP document server answers a snapshot its reliable queue cannot take", async () => { + // Two documents each served in exactly the eight fragments the queue holds are admitted; a + // snapshot of both needs sixteen and cannot be queued. The peer is answered with a + // cultnet.error.v0, not left to time out, and onError hears why. + const atBound = bodyLengthForReplyBytes("note:q1", 8 * 1024); + await withServedBoundRig(0x10203068, { maxFragmentBytes: 1024, maxPendingReliablePackets: 8 }, async (rig) => { + rig.peer.send(notePut("put-q1", "note:q1", atBound)); + rig.peer.send(notePut("put-q2", "note:q2", atBound)); + await waitForAdmission(rig, "note:q1"); + await waitForAdmission(rig, "note:q2"); + assert.deepEqual(rig.errors, []); + rig.peer.sendSnapshotRequest({ schemaVersion: "cultnet.snapshot_request.v0", messageId: "both", recordKeys: ["note:q1", "note:q2"] }); + await waitForPeerErrors(rig, 1); + assert.equal(rig.errors.length, 1); + assert.ok(!rig.messages.some((message) => message.schemaVersion === "cultnet.snapshot_response_raw.v0")); + }); +}); + test("CultMesh TS RUDP document server honours maxPayloadBytes for replies and judges puts by what it would serve", async () => { const limit = 4_000; const atBound = bodyLengthForReplyBytes("note:at", limit); @@ -3576,17 +3632,6 @@ async function waitForPeerErrors(rig: ServedBoundRig, count: number): Promise error.message).join("; ")}`); } -test("CultMesh TS RUDP document server reports a datagram that fails to leave", async () => { - // Fragments larger than any UDP datagram: the reply's one packet cannot be sent. - await withServedBoundRig(0x10203063, { maxFragmentBytes: 100_000, maxPendingReliablePackets: 512 }, async (rig) => { - rig.peer.send(notePut("put-large", "note:large", 80_000)); - await waitForAdmission(rig, "note:large"); - rig.peer.sendSnapshotRequest({ schemaVersion: "cultnet.snapshot_request.v0", messageId: "0", recordKeys: ["note:large"] }); - const failure = await waitForError(rig, "the oversized datagram"); - assert.match(failure.message, /EMSGSIZE|message too long/i); - }); -}); - function delay(milliseconds: number): Promise { return new Promise(resolve => setTimeout(resolve, milliseconds)); } diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts new file mode 100644 index 00000000..64f19111 --- /dev/null +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -0,0 +1,221 @@ +// A datagram the document server cannot send to one peer is that peer's problem +// alone. It never escapes a timer or the receive handler, never stops the server +// serving the others, and only a datagram that can never be sent as built ends +// the session that owes it. +import assert from "node:assert/strict"; +import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode } from "@msgpack/msgpack"; +import { + CultNetDocumentRegistry, + CultNetRudpSession, + decodeRudpPacket, + encodeRudpPacket, + type CultNetPeer, +} from "cultnet-ts"; +import { CultMesh } from "../src/index"; + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 3_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +function put(recordKey: string) { + return { + schemaVersion: "cultnet.document_put_raw.v0" as const, + messageId: `put-${recordKey}`, + document: { + schemaId: "cultmesh.note.v0", + recordKey, + storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encode(recordKey), + }, + }; +} + +interface Server { + socket: Socket; + server: ReturnType; + /** The record key of every put received, with the port it came from. */ + puts: Map; + closed: number[]; + errors: Error[]; + peer(name: string): Promise; +} + +async function startServer(connectionId: number, socket: Socket): Promise { + const puts = new Map(); + const closed: number[] = []; + const errors: Error[] = []; + const server = CultMesh.createRudpDocumentServer("send-failure-server", connectionId, { + documents: new CultNetDocumentRegistry(), + socket, + bindHost: "127.0.0.1", + bindPort: 0, + resendDelayMs: 10, + resendPollMs: 10, + sessionTimeoutMs: 1_000, + onDocumentPutRaw: (document) => { + puts.set(document.recordKey, document.remote.port); + }, + onSessionClosed: (session) => { + closed.push(session.remote.port); + }, + onError: (error) => errors.push(error), + }); + await server.start(); + const peer = (name: string) => + CultMesh.createRudpPeer(name, connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 10, + resendPollMs: 5, + connectTimeoutMs: 2_000, + }); + return { socket, server, puts, closed, errors, peer }; +} + +test("a Connect whose Accept can never be sent starts no session, and the server keeps serving", async () => { + const connectionId = 0x10203091; + const uncaught = watchUncaught(); + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + let peer: CultNetPeer | undefined; + try { + // A Connect as it arrives from UDP source port 0, which Node refuses to answer. + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); + // Past the idle timeout: a session that had started would have been closed by now. + await sleep(1_300); + + peer = await served.peer("after-port-zero"); + peer.send(put("after")); + await waitFor(() => served.puts.has("after"), "the real peer's put"); + assert.deepEqual(served.closed, [], "no session started for port 0"); + assert.deepEqual(served.errors.map((error) => error.message), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + peer?.close(); + served.server.close(); + } +}); + +test("a datagram that may yet pass is counted; one that never can ends only that peer's session", async () => { + const connectionId = 0x10203092; + const socket = createSocket("udp4"); + // Sends to a failing port report the failure the way Node reports a system one: + // later, through the send's callback. + const failing = new Map(); + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + const code = failing.get(port); + if (code === undefined) { + send(message, port, address, callback); + return; + } + process.nextTick(() => callback?.(Object.assign(new Error(`send failed: ${code}`), { code }))); + }, + }); + const served = await startServer(connectionId, socket); + let x: CultNetPeer | undefined; + let y: CultNetPeer | undefined; + try { + x = await served.peer("x"); + y = await served.peer("y"); + x.send(put("x-1")); + await waitFor(() => served.puts.has("x-1"), "X's first put"); + const xPort = served.puts.get("x-1")!; + + // No route: X's acknowledgements are lost datagrams, counted, never reported as errors, + // and end nothing. + assert.equal(served.server.sendFailures, 0); + failing.set(xPort, "EHOSTUNREACH"); + x.send(put("x-2")); + await waitFor(() => served.server.sendFailures >= 2, "the counted losses"); + y.send(put("y-1")); + await waitFor(() => served.puts.has("y-1"), "Y's put"); + assert.deepEqual(served.closed, [], "a loss that may pass ends no session"); + assert.deepEqual(served.errors.map((error) => error.message), [], "a lost datagram is not an error"); + + // X's retransmitted put is acknowledged again, and that datagram can never be sent. + failing.set(xPort, "EMSGSIZE"); + await waitFor(() => served.closed.length > 0, "X's session to end"); + assert.deepEqual(served.closed, [xPort]); + assert.equal( + served.errors.some((error) => (error as NodeJS.ErrnoException).code === "EMSGSIZE"), + false, + "a datagram that can never be sent ends the session instead of being reported", + ); + + y.send(put("y-2")); + await waitFor(() => served.puts.has("y-2"), "Y's put after X ended"); + assert.deepEqual(served.closed, [xPort]); + } finally { + x?.close(); + y?.close(); + served.server.close(); + } +}); + +test("a packet the session refuses ends only that peer's session, and the peer is told", async () => { + const connectionId = 0x10203093; + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + const raw = createSocket("udp4"); + await new Promise((resolve) => raw.bind(0, "127.0.0.1", resolve)); + const arrived: ReturnType[] = []; + raw.on("message", (wire) => arrived.push(decodeRudpPacket(wire))); + let y: CultNetPeer | undefined; + try { + const session = new CultNetRudpSession({ connectionId }); + raw.send(encodeRudpPacket(session.createConnect(Date.now())), served.server.bind.port, "127.0.0.1"); + await waitFor(() => arrived.some((packet) => packet.packetType === "accept"), "the Accept"); + session.receive(arrived.find((packet) => packet.packetType === "accept")!, Date.now()); + y = await served.peer("y"); + + // A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + // The session records its sequence before it refuses it. + const [fragment] = session.sendMany("schema", new Uint8Array([1]), { reliable: true, ordered: true }); + raw.send( + encodeRudpPacket({ ...fragment!, fragmentId: 0, fragmentIndex: 0, fragmentCount: 2 }), + served.server.bind.port, + "127.0.0.1", + ); + await waitFor(() => arrived.some((packet) => packet.packetType === "disconnect"), "the goodbye"); + const goodbye = arrived.find((packet) => packet.packetType === "disconnect")!; + assert.equal(Buffer.from(goodbye.payload ?? []).toString("utf8"), "session refused a packet"); + // The goodbye's ack field does not acknowledge the refused frame, or the peer would + // count as delivered a frame that was never handled. + const acknowledged = + goodbye.ack === fragment!.sequence || + (goodbye.ack > fragment!.sequence && + goodbye.ack - fragment!.sequence <= 32 && + (goodbye.ackMask & (1 << (goodbye.ack - fragment!.sequence - 1))) !== 0); + assert.equal(acknowledged, false, "the goodbye acknowledged the refused frame"); + assert.deepEqual(served.closed, [(raw.address() as { port: number }).port]); + + y.send(put("y-after")); + await waitFor(() => served.puts.has("y-after"), "Y's put"); + assert.deepEqual(served.errors.map((error) => error.message), []); + } finally { + y?.close(); + raw.close(); + served.server.close(); + } +}); diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index b1734f25..3344f438 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -6,6 +6,7 @@ from .client import ( CultNetSchemaTransportFactory, create_rudp_schema_transport, create_tcp_framed_schema_transport, + rudp_client_bind_host, ) from .cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, @@ -122,6 +123,7 @@ from .transport import ( TcpFramedTransportConnection, compute_reconnect_delay_ms, create_reconnect_policy, + is_permanent_send_error, create_rudp_transport_profile, create_tcp_framed_transport_profile, decode_rudp_packet, @@ -208,6 +210,7 @@ __all__ = [ "CultNetRudpReceiveResult", "CultNetRudpSendOptions", "CultNetRudpSession", + "is_permanent_send_error", "CultNetRudpSessionOptions", "CultNetRudpSocketMode", "CultNetRudpReconnectLoop", @@ -228,6 +231,7 @@ __all__ = [ "compute_reconnect_delay_ms", "create_reconnect_policy", "create_rudp_schema_transport", + "rudp_client_bind_host", "create_tcp_framed_schema_transport", "create_rudp_transport_profile", "create_tcp_framed_transport_profile", diff --git a/packages/cultnet-py/src/cultnet_py/client.py b/packages/cultnet-py/src/cultnet_py/client.py index 05bc5fa6..28060753 100644 --- a/packages/cultnet-py/src/cultnet_py/client.py +++ b/packages/cultnet-py/src/cultnet_py/client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ipaddress import socket import time from collections.abc import Callable @@ -384,6 +385,22 @@ def create_tcp_framed_schema_transport( ) +def rudp_client_bind_host(endpoint_host: str) -> str: + """Return the address an RUDP client binds when its caller names none. + + Loopback for a loopback endpoint, otherwise the unspecified address of the + endpoint's family. A socket bound to loopback cannot send off the host on + Windows. + """ + family, _, _, _, address = socket.getaddrinfo( + endpoint_host.removeprefix("[").removesuffix("]"), None, type=socket.SOCK_DGRAM + )[0] + loopback = ipaddress.ip_address(address[0]).is_loopback + if family == socket.AF_INET6: + return "::1" if loopback else "::" + return "127.0.0.1" if loopback else "0.0.0.0" + + def create_rudp_schema_transport( *, host: str, @@ -391,7 +408,7 @@ def create_rudp_schema_transport( connection_id: int, timeout_seconds: float = 4.0, runtime_id: str = "cultnet-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, initial_sequence: int | None = None, resend_delay_ms: int = 25, @@ -400,7 +417,11 @@ def create_rudp_schema_transport( max_fragment_bytes: int | None = None, max_pending_reliable_packets: int | None = None, ) -> CultNetRudpSocketTransportConnection: - transport_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if bind_host is None: + bind_host = rudp_client_bind_host(host) + transport_socket = socket.socket( + socket.AF_INET6 if ":" in bind_host else socket.AF_INET, socket.SOCK_DGRAM + ) transport_socket.bind((bind_host, bind_port)) transport_socket.settimeout(min(timeout_seconds, 0.02)) transport = CultNetRudpSocketTransportConnection( diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 355faa51..78ae9542 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import secrets import socket import threading @@ -435,9 +436,39 @@ def create_rudp_transport_profile( } +UNSENDABLE_PACKET_REASON = "packet could not be sent" + +_PERMANENT_SEND_ERRNOS = frozenset( + code + for code in ( + errno.EMSGSIZE, + errno.EINVAL, + errno.EAFNOSUPPORT, + getattr(errno, "WSAEMSGSIZE", None), + getattr(errno, "WSAEINVAL", None), + getattr(errno, "WSAEAFNOSUPPORT", None), + ) + if code is not None +) + + +def is_permanent_send_error(error: OSError) -> bool: + """Whether a failed `sendto` can never succeed for the datagram as built, so resending + it is pointless: it is too large for a datagram (EMSGSIZE), its address is malformed for + the socket (EINVAL), or belongs to another address family (EAFNOSUPPORT), with their + Winsock equivalents. Everything else (no route, full buffers, firewall drops, a refusal + reported for an earlier datagram, an interrupted call) is a property of the path or the + moment and may pass, so it is a lost datagram, not a reason to end a session.""" + return error.errno in _PERMANENT_SEND_ERRNOS + + class CultNetRudpSession: RELIABLE_SEND_WINDOW_PACKETS = 32 RECEIVED_SEQUENCE_WINDOW = 4_096 + # Flow window: a reliable packet is admitted only while its sequence is at most FLOW_WINDOW_SEQUENCES above the + # lowest unacked one and the payload above that sequence stays within FLOW_WINDOW_BYTES. + FLOW_WINDOW_SEQUENCES = 1_023 + FLOW_WINDOW_BYTES = 4 * 1024 * 1024 def __init__(self, options: CultNetRudpSessionOptions) -> None: self.connection_id = _uint32(options.connection_id, "connection_id") @@ -478,6 +509,9 @@ class CultNetRudpSession: self._received_sequences: set[int] = set() self._latest_sequenced_by_channel: dict[str, int] = {} self._pending_reliable: dict[int, _PendingReliablePacket] = {} + # Payload sizes of acknowledged reliable sequences above the lowest unacknowledged one. The receiver still + # holds those bytes behind the gap, so the flow window keeps counting them until the lowest sequence passes them. + self._acked_above_lowest: dict[int, int] = {} self._queued_reliable: deque[CultNetRudpPacket] = deque() # Every reliable sequence up to and including this one has been received # since the peer state was last reset. Only the handshake seeds it: the @@ -540,6 +574,7 @@ class CultNetRudpSession: self._awaiting_accept = False self._generation += 1 self._pending_reliable.clear() + self._acked_above_lowest.clear() self._queued_reliable.clear() def reset_peer_state(self) -> None: @@ -656,7 +691,7 @@ class CultNetRudpSession: options: CultNetRudpSendOptions | None = None, ) -> CultNetRudpPacket: resolved = options or CultNetRudpSendOptions() - if resolved.reliable and len(self._pending_reliable) >= self.RELIABLE_SEND_WINDOW_PACKETS: + if resolved.reliable and (self._queued_reliable or not self._window_admits(self._next_sequence, len(payload))): raise ValueError("RUDP reliable send window is full; receive acknowledgements before sending") return self.send_many(channel_id, payload, resolved)[0] @@ -833,6 +868,14 @@ class CultNetRudpSession: def create_ping(self, payload: bytes = b"") -> CultNetRudpPacket: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) + def end_unsendable_session(self, error: OSError) -> CultNetRudpPacket: + """Ends a session that owes its peer a packet that can never be sent as built + (see `is_permanent_send_error`). Resending it would fail forever, and dropping it + would leave a reliable sequence the peer waits on for good, so the session cannot + be kept. Returns the goodbye for the peer; its reason names the error.""" + self.reset_peer_state() + return self.create_disconnect(f"{UNSENDABLE_PACKET_REASON}: {error}".encode("utf-8")) + def create_disconnect(self, reason: bytes = b"") -> CultNetRudpPacket: self._end_session() return self._create_packet(CultNetRudpPacketType.DISCONNECT, "control", reason) @@ -935,22 +978,39 @@ class CultNetRudpSession: last_sent_at_ms=now_ms, ) + def _window_admits(self, sequence: int, payload_length: int) -> bool: + """Whether a reliable packet may go on the wire now: the window has a slot, and its sequence and the + payload above the lowest unacked sequence stay inside the flow window. With nothing pending, any packet + is admissible.""" + if len(self._pending_reliable) >= self.RELIABLE_SEND_WINDOW_PACKETS: + return False + if not self._pending_reliable: + return True + lowest = min(self._pending_reliable) + bytes_above = sum( + len(pending.packet.payload) for pending_sequence, pending in self._pending_reliable.items() if pending_sequence > lowest + ) + sum(self._acked_above_lowest.values()) + return sequence - lowest <= self.FLOW_WINDOW_SEQUENCES and bytes_above + payload_length <= self.FLOW_WINDOW_BYTES + def _admit_reliable_packets( self, packets: tuple[CultNetRudpPacket, ...], now_ms: int, ) -> tuple[CultNetRudpPacket, ...]: - available = max(0, self.RELIABLE_SEND_WINDOW_PACKETS - len(self._pending_reliable)) - ready = packets[:available] - for packet in ready: - self._track_reliable(packet, now_ms) - self._queued_reliable.extend(packets[available:]) - return ready + ready: list[CultNetRudpPacket] = [] + for packet in packets: + if not self._queued_reliable and self._window_admits(packet.sequence, len(packet.payload)): + self._track_reliable(packet, now_ms) + ready.append(packet) + else: + self._queued_reliable.append(packet) + return tuple(ready) def _promote_queued_reliable(self, now_ms: int) -> tuple[CultNetRudpPacket, ...]: - available = max(0, self.RELIABLE_SEND_WINDOW_PACKETS - len(self._pending_reliable)) ready: list[CultNetRudpPacket] = [] - while len(ready) < available and self._queued_reliable: + while self._queued_reliable and self._window_admits( + self._queued_reliable[0].sequence, len(self._queued_reliable[0].payload) + ): packet = self._queued_reliable.popleft() self._track_reliable(packet, now_ms) ready.append(packet) @@ -963,10 +1023,21 @@ class CultNetRudpSession: raise ValueError("RUDP reliable send queue is full") def _apply_acknowledgements(self, packet: CultNetRudpPacket) -> None: - self._pending_reliable.pop(packet.ack, None) + self._acknowledge(packet.ack) for bit in range(32): if packet.ack_mask & (1 << bit): - self._pending_reliable.pop(packet.ack - bit - 1, None) + self._acknowledge(packet.ack - bit - 1) + # Acknowledged sizes stop counting once the lowest unacknowledged sequence passes them, and all of them stop + # when nothing is unacknowledged. + lowest = min(self._pending_reliable, default=None) + self._acked_above_lowest = ( + {} if lowest is None else {sequence: length for sequence, length in self._acked_above_lowest.items() if sequence > lowest} + ) + + def _acknowledge(self, sequence: int) -> None: + pending = self._pending_reliable.pop(sequence, None) + if pending is not None: + self._acked_above_lowest[sequence] = len(pending.packet.payload) def _seed_received(self, sequence: int) -> None: """The handshake's one act on the watermark: the peer's Connect or Accept diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index fb2eff54..765f5c37 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -30,6 +30,8 @@ from cultnet_py.interop_peer import ( ) from cultnet_py import ( compute_simulation_claim_hash, + create_rudp_schema_transport, + rudp_client_bind_host, CultNetDatabaseChange, CultNetRawClient, CultNetRawDocumentRecord, @@ -877,6 +879,185 @@ class CultNetTests(unittest.TestCase): self.assertEqual(old_ack.ack, oldest_sequence) self.assertEqual(old_ack.ack_mask, 0) + def test_cultnet_rudp_only_a_datagram_that_can_never_be_sent_is_a_permanent_failure(self) -> None: + import errno + + from cultnet_py import is_permanent_send_error + + for code in (errno.EMSGSIZE, errno.EINVAL, errno.EAFNOSUPPORT): + self.assertTrue(is_permanent_send_error(OSError(code, "permanent")), errno.errorcode[code]) + for code in ( + errno.EPERM, + errno.EINTR, + errno.EAGAIN, + errno.EACCES, + errno.ENETDOWN, + errno.ENETUNREACH, + errno.EADDRNOTAVAIL, + errno.ENOBUFS, + errno.ECONNREFUSED, + errno.EHOSTUNREACH, + ): + self.assertFalse(is_permanent_send_error(OSError(code, "transient")), errno.errorcode[code]) + self.assertFalse(is_permanent_send_error(OSError("no errno"))) + + # The sender's flow window: a reliable packet goes on the wire only while its sequence is at most 1,023 above + # the lowest unacked one and the payload above that sequence stays within 4 MiB. + FLOW_CONNECTION_ID = 0x464C4F57 + MIB = 1024 * 1024 + + def _flow_session(self, initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=self.FLOW_CONNECTION_ID, initial_sequence=initial_sequence) + ) + + def _connected_flow_pair( + self, sender_initial_sequence: int, receiver_initial_sequence: int + ) -> tuple[CultNetRudpSession, CultNetRudpSession]: + """A sender and a receiver that have shaken hands: the receiver's watermark is seeded by the sender's + Connect, and the sender's Connect is acknowledged, so nothing is pending on the sender.""" + sender = self._flow_session(sender_initial_sequence) + receiver = self._flow_session(receiver_initial_sequence) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) + return sender, receiver + + def _connected_flow_session(self, initial_sequence: int) -> CultNetRudpSession: + return self._connected_flow_pair(initial_sequence, 900)[0] + + @staticmethod + def _flow_send(session: CultNetRudpSession, payload_bytes: int) -> tuple[CultNetRudpPacket, ...]: + return session.send_many("state", bytes(payload_bytes), CultNetRudpSendOptions(reliable=True)) + + def _flow_ack(self, sequence: int) -> CultNetRudpPacket: + return CultNetRudpPacket(CultNetRudpPacketType.ACK, self.FLOW_CONNECTION_ID, 0, sequence, 0, "control") + + def _lose_g_and_fill_the_span(self, session: CultNetRudpSession) -> CultNetRudpPacket: + g = self._flow_send(session, 1)[0] + for offset in range(1, 1024): + self.assertEqual([p.sequence for p in self._flow_send(session, 1)], [g.sequence + offset]) + session.receive(self._flow_ack(g.sequence + offset), 1) + return g + + def test_cultnet_rudp_a_lost_packet_holds_the_sender_1023_sequences_ahead_and_is_still_delivered(self) -> None: + sender, receiver = self._connected_flow_pair(1, 100) + g = self._flow_send(sender, 1)[0] + admitted_after_g = 0 + for _ in range(4200): + for packet in self._flow_send(sender, 1): + admitted_after_g += 1 + receiver.receive(packet, 1) + sender.receive(receiver.create_ack(), 1) + self.assertEqual(admitted_after_g, 1023) + self.assertEqual(sender.queued_reliable_packet_count, 4200 - 1023) + + retransmit = next(p for p in sender.due_resends(1000) if p.sequence == g.sequence) + delivered = receiver.receive(retransmit, 1000).delivered + self.assertEqual([frame.sequence for frame in delivered], [g.sequence]) + + promoted = sender.receive(receiver.create_ack_for_received(g.sequence), 1001).ready_to_send + self.assertEqual(promoted[0].sequence, g.sequence + 1024) + + def test_cultnet_rudp_a_direct_send_past_the_span_is_refused_without_consuming_a_sequence(self) -> None: + sender = self._connected_flow_session(1) + g = self._lose_g_and_fill_the_span(sender) + with self.assertRaises(ValueError): + sender.send("state", bytes(1), CultNetRudpSendOptions(reliable=True)) + self.assertEqual(sender.queued_reliable_packet_count, 0) + sender.receive(self._flow_ack(g.sequence), 2) + following = sender.send("state", bytes(1), CultNetRudpSendOptions(reliable=True)) + self.assertEqual(following.sequence, g.sequence + 1024) + + def test_cultnet_rudp_payload_above_the_lowest_unacked_sequence_is_bounded_by_4_mib(self) -> None: + sender = self._connected_flow_session(1) + g = self._flow_send(sender, self.MIB)[0] + # The lowest unacked packet's own bytes do not count: four more MiB fit above it. + for _ in range(4): + self.assertEqual(len(self._flow_send(sender, self.MIB)), 1) + self.assertEqual(len(self._flow_send(sender, self.MIB)), 0) + self.assertEqual(len(self._flow_send(sender, self.MIB)), 0) + self.assertEqual(sender.queued_reliable_packet_count, 2) + + # Acking g moves the floor up one packet: exactly one MiB more fits. + promoted = sender.receive(self._flow_ack(g.sequence), 1).ready_to_send + self.assertEqual([p.sequence for p in promoted], [g.sequence + 5]) + self.assertEqual(sender.queued_reliable_packet_count, 1) + + def test_cultnet_rudp_a_packet_larger_than_the_window_goes_out_alone_and_waits_behind_anything_unacked(self) -> None: + self.assertEqual(len(self._flow_send(self._connected_flow_session(1), 5 * self.MIB)), 1) + sender = self._connected_flow_session(1) + g = self._flow_send(sender, 1)[0] + self.assertEqual(len(self._flow_send(sender, 5 * self.MIB)), 0) + promoted = sender.receive(self._flow_ack(g.sequence), 1).ready_to_send + self.assertEqual([p.sequence for p in promoted], [g.sequence + 1]) + + def test_cultnet_rudp_a_small_packet_does_not_overtake_a_queued_large_one(self) -> None: + sender = self._connected_flow_session(1) + g = self._flow_send(sender, 1)[0] + self.assertEqual(len(self._flow_send(sender, 3 * self.MIB)), 1) + self.assertEqual(len(self._flow_send(sender, 2 * self.MIB)), 0) + # A direct send refuses instead of queueing behind it, and takes no sequence. + with self.assertRaises(ValueError): + sender.send("state", bytes(1), CultNetRudpSendOptions(reliable=True)) + self.assertEqual(sender.queued_reliable_packet_count, 1) + # One byte would fit above g, but the 2 MiB packet is ahead of it. + self.assertEqual(len(self._flow_send(sender, 1)), 0) + promoted = sender.receive(self._flow_ack(g.sequence), 1).ready_to_send + self.assertEqual([p.sequence for p in promoted], [g.sequence + 2, g.sequence + 3]) + + def test_cultnet_rudp_acknowledged_bytes_above_a_lost_packet_still_count_against_4_mib(self) -> None: + sender = self._connected_flow_session(1) + g = self._flow_send(sender, 1)[0] + # g is lost. Each 1 MiB frame above it is delivered and acknowledged, and the receiver still holds it behind + # the gap, so it keeps counting. + for offset in range(1, 5): + self.assertEqual([p.sequence for p in self._flow_send(sender, self.MIB)], [g.sequence + offset]) + sender.receive(self._flow_ack(g.sequence + offset), 1) + self.assertEqual(len(self._flow_send(sender, self.MIB)), 0) + self.assertEqual(sender.queued_reliable_packet_count, 1) + with self.assertRaises(ValueError): + sender.send("state", bytes(self.MIB), CultNetRudpSendOptions(reliable=True)) + + # g arrives: nothing is held behind a gap any more, and the queue drains. + promoted = sender.receive(self._flow_ack(g.sequence), 2).ready_to_send + self.assertEqual([p.sequence for p in promoted], [g.sequence + 5]) + # The bytes above the old gap no longer count: another MiB fits. + self.assertEqual([p.sequence for p in self._flow_send(sender, self.MIB)], [g.sequence + 6]) + + def test_cultnet_rudp_acknowledged_bytes_count_against_the_lowest_unacked_sequence_as_it_advances(self) -> None: + sender = self._connected_flow_session(1) + g = self._flow_send(sender, 1)[0] + h = self._flow_send(sender, 1)[0] + for _ in range(3): + self._flow_send(sender, self.MIB) + # g and the 3 MiB above h are acknowledged; h is lost. + for offset in (2, 3, 4, 0): + sender.receive(self._flow_ack(g.sequence + offset), 1) + self.assertEqual([p.sequence for p in self._flow_send(sender, self.MIB)], [g.sequence + 5]) + # 4 MiB sit above h, acknowledged or not: one more byte does not fit. + self.assertEqual(len(self._flow_send(sender, 1)), 0) + # h arrives, the floor moves to g + 5, and the byte fits. + promoted = sender.receive(self._flow_ack(h.sequence), 2).ready_to_send + self.assertEqual([p.sequence for p in promoted], [g.sequence + 6]) + + def test_cultnet_rudp_a_new_generation_on_the_server_does_not_inherit_acknowledged_bytes(self) -> None: + # A server that accepts a fresh Connect may send before its Accept is acknowledged, so no acknowledgement of + # the new generation has moved the floor yet. Bytes the ended generation had acknowledged above its own lost + # packet belong to no receiver's hold any more and must not count. + server = self._flow_session(500) + accept = server.accept_connect(self._flow_session(1).create_connect(0), 0) + server.receive(self._flow_ack(accept.sequence), 0) + g = self._flow_send(server, 1)[0] + for offset in range(1, 5): + self.assertEqual([p.sequence for p in self._flow_send(server, self.MIB)], [g.sequence + offset]) + server.receive(self._flow_ack(g.sequence + offset), 1) + self.assertEqual(len(self._flow_send(server, self.MIB)), 0) + + next_accept = server.accept_connect(self._flow_session(200_000).create_connect(2), 2) + self.assertEqual(next_accept.packet_type, CultNetRudpPacketType.ACCEPT) + # The Accept is unacknowledged and the only thing pending: 1 MiB fits above it. + sent = server.send("state", bytes(self.MIB), CultNetRudpSendOptions(reliable=True)) + self.assertEqual(sent.sequence, next_accept.sequence + 1) + def test_cultnet_rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames(self) -> None: server_socket = bind_udp_socket() client_socket = bind_udp_socket() @@ -2399,6 +2580,80 @@ class CultNetTests(unittest.TestCase): self.assertEqual(CultNetSimulationConsensusCandidate.from_wire(candidates[0]), typed_candidates[0]) +def _non_loopback_ipv4_address() -> str | None: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.connect(("192.0.2.1", 9)) + address = probe.getsockname()[0] + except OSError: + return None + finally: + probe.close() + return None if address.startswith("127.") or address == "0.0.0.0" else address + + +class CultNetRudpClientBindTests(unittest.TestCase): + def test_an_rudp_client_binds_loopback_only_for_a_loopback_endpoint(self) -> None: + self.assertEqual(rudp_client_bind_host("10.77.0.1"), "0.0.0.0") + self.assertEqual(rudp_client_bind_host("2001:db8::1"), "::") + self.assertEqual(rudp_client_bind_host("[2001:db8::1]"), "::") + self.assertEqual(rudp_client_bind_host("127.0.0.1"), "127.0.0.1") + self.assertEqual(rudp_client_bind_host("127.0.0.2"), "127.0.0.1") + self.assertEqual(rudp_client_bind_host("::1"), "::1") + self.assertEqual(rudp_client_bind_host("[::1]"), "::1") + + # A socket bound to loopback cannot send off the host on Windows. On Linux + # it reaches a local non-loopback address, but from 127.0.0.1, so the + # source address shows which one the client bound. + def test_a_schema_transport_for_a_non_loopback_endpoint_does_not_send_from_loopback(self) -> None: + address = _non_loopback_ipv4_address() + if address is None: + self.skipTest("no non-loopback IPv4 address on this host") + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("0.0.0.0", 0)) + listener.settimeout(2.0) + try: + with self.assertRaises(TimeoutError): + create_rudp_schema_transport( + host=address, + port=listener.getsockname()[1], + connection_id=0x43554C54, + timeout_seconds=0.2, + ) + _, source = listener.recvfrom(65535) + self.assertEqual(source[0], address) + finally: + listener.close() + + # The endpoint alone would bind 127.0.0.1; the explicit host must win, and the + # source address shows which one was bound. It stays on loopback, because a + # loopback-bound socket cannot send off the host on Windows. + def test_a_schema_transport_binds_an_explicit_bind_host(self) -> None: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.bind(("127.0.0.2", 0)) + except OSError: + self.skipTest("127.0.0.2 is not a local address on this host") + finally: + probe.close() + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("127.0.0.1", 0)) + listener.settimeout(2.0) + try: + with self.assertRaises(TimeoutError): + create_rudp_schema_transport( + host="127.0.0.1", + port=listener.getsockname()[1], + connection_id=0x43554C54, + timeout_seconds=0.2, + bind_host="127.0.0.2", + ) + _, source = listener.recvfrom(65535) + self.assertEqual(source[0], "127.0.0.2") + finally: + listener.close() + + class CultNetRudpFragmentBoundTests(unittest.TestCase): def _pair(self) -> tuple[CultNetRudpSession, CultNetRudpSession]: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=1)) diff --git a/packages/cultnet-rs/Cargo.toml b/packages/cultnet-rs/Cargo.toml index b1cd8d11..f87af6cb 100644 --- a/packages/cultnet-rs/Cargo.toml +++ b/packages/cultnet-rs/Cargo.toml @@ -19,7 +19,6 @@ cultcache-rs = { path = "../cultcache-rs" } ed25519-dalek = "2.2" hmac = "0.12" rand = "0.9" -reed-solomon-erasure = "6" rmp-serde = "1.3" rmpv = { version = "1.3", features = ["with-serde"] } serde = { version = "1", features = ["derive"] } diff --git a/packages/cultnet-rs/src/cultmesh.rs b/packages/cultnet-rs/src/cultmesh.rs index 900f4744..920a9f1f 100644 --- a/packages/cultnet-rs/src/cultmesh.rs +++ b/packages/cultnet-rs/src/cultmesh.rs @@ -3,6 +3,9 @@ use anyhow::anyhow; use chrono::DateTime; use chrono::Utc; use std::collections::BTreeMap; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::net::SocketAddr; use std::net::ToSocketAddrs; use std::net::UdpSocket; @@ -240,7 +243,10 @@ impl CultNetRudpEndpoint { #[derive(Clone, Debug)] pub struct CultMeshRudpSocketOptions { - pub bind_host: String, + /// Local address to bind. Unset, a server binds loopback, and a client binds + /// loopback for a loopback endpoint and the unspecified address of the + /// endpoint's family otherwise, so it can reach a remote host. + pub bind_host: Option, pub bind_port: u16, pub read_timeout: Option, pub initial_sequence: Option, @@ -253,7 +259,7 @@ pub struct CultMeshRudpSocketOptions { impl Default for CultMeshRudpSocketOptions { fn default() -> Self { Self { - bind_host: "127.0.0.1".to_string(), + bind_host: None, bind_port: 0, read_timeout: Some(Duration::from_millis(20)), initial_sequence: None, @@ -316,7 +322,8 @@ impl CultMesh { connection_id: u32, options: CultMeshRudpSocketOptions, ) -> Result { - let socket = bind_rudp_socket(&options)?; + let bind_host = options.bind_host.as_deref().unwrap_or("127.0.0.1"); + let socket = bind_rudp_socket(bind_host, &options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_reliable_expire_after_ms: None, media_delivery: None, @@ -341,14 +348,19 @@ impl CultMesh { endpoint: &CultNetRudpEndpoint, options: CultMeshRudpSocketOptions, ) -> Result { - let socket = bind_rudp_socket(&options)?; + let remote_addr = endpoint.socket_addr()?; + let bind_host = match options.bind_host.as_deref() { + Some(host) => host.to_string(), + None => client_bind_ip(remote_addr).to_string(), + }; + let socket = bind_rudp_socket(&bind_host, &options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_reliable_expire_after_ms: None, media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Client, - remote_addr: Some(endpoint.socket_addr()?), + remote_addr: Some(remote_addr), connection_id, initial_sequence: options.initial_sequence, resend_delay_ms: options.resend_delay_ms, @@ -503,10 +515,21 @@ pub fn cultnet_rudp_endpoint(endpoint: &str) -> Result { Ok(CultNetRudpEndpoint { host, port }) } -fn bind_rudp_socket(options: &CultMeshRudpSocketOptions) -> Result { +/// A socket bound to loopback cannot send off the host on Windows, so a client +/// keeps loopback only when its peer is on loopback. +fn client_bind_ip(remote_addr: SocketAddr) -> IpAddr { + match (remote_addr.ip().is_loopback(), remote_addr) { + (true, SocketAddr::V4(_)) => IpAddr::V4(Ipv4Addr::LOCALHOST), + (true, SocketAddr::V6(_)) => IpAddr::V6(Ipv6Addr::LOCALHOST), + (false, SocketAddr::V4(_)) => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + (false, SocketAddr::V6(_)) => IpAddr::V6(Ipv6Addr::UNSPECIFIED), + } +} + +fn bind_rudp_socket(bind_host: &str, options: &CultMeshRudpSocketOptions) -> Result { let socket = UdpSocket::bind(format!( "{}:{}", - socket_host(&options.bind_host), + socket_host(bind_host), options.bind_port ))?; socket.set_read_timeout(options.read_timeout)?; diff --git a/packages/cultnet-rs/src/media_fec.rs b/packages/cultnet-rs/src/media_fec.rs index 490eaaed..38525db5 100644 --- a/packages/cultnet-rs/src/media_fec.rs +++ b/packages/cultnet-rs/src/media_fec.rs @@ -14,11 +14,11 @@ //! the `k + m` shards recover the block (the code is MDS), so any `m` erasures //! are repaired and any more are reported as [`MediaFecError::BeyondRepair`], //! never as wrong bytes. The construction is named on every parity record by -//! [`MEDIA_FEC_SCHEME_RS_GF256_V1`] and backed by the `reed-solomon-erasure` -//! crate. The library is not the contract: the construction below is, and -//! `tests/media_fec.rs` pins the parity bytes of known inputs, so a library that -//! changes the matrix fails there instead of on the wire, and a decoder in -//! another runtime has vectors to meet. +//! [`MEDIA_FEC_SCHEME_RS_GF256_V1`] and implemented in this module, from the +//! text below. The construction is the contract: `tests/media_fec.rs` and +//! `tests/fixtures/media_fec_rs_gf256_v1.kat` pin the parity bytes of known +//! inputs, so a change to the math fails there instead of on the wire, and a +//! decoder in another runtime has vectors to meet. //! //! # The `rs-gf256-v1` construction //! @@ -42,8 +42,10 @@ //! * Any `k` surviving shards recover the block: invert the `k x k` submatrix //! of `E` for the surviving rows and multiply. //! -//! The known-answer vectors in `tests/media_fec.rs` were checked against an -//! independent implementation of exactly this text. +//! The known-answer vectors were produced by `reed-solomon-erasure` 6.0.0's +//! `galois_8` field, which builds exactly this matrix, before this module +//! replaced it; the smaller ones in `tests/media_fec.rs` were also checked +//! against a second, independent implementation of this text. //! //! # What a block is //! @@ -88,8 +90,6 @@ use std::collections::BTreeMap; use std::fmt; -use reed_solomon_erasure::galois_8::ReedSolomon; - use crate::media_stream_contracts::{ GameCultMediaAudioPacketRecord, GameCultMediaAudioParityShardRecord, GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, @@ -102,8 +102,8 @@ use crate::media_stream_wire::{ use crate::rudp::RUDP_FIXED_HEADER_BYTES; /// Names the construction on every parity record: systematic Reed-Solomon over -/// GF(2^8), as built by `reed-solomon-erasure`'s `galois_8` field. A different -/// matrix is a different scheme id. +/// GF(2^8), as the module docs state it. A different matrix is a different +/// scheme id. pub const MEDIA_FEC_SCHEME_RS_GF256_V1: &str = "rs-gf256-v1"; /// The largest UDP payload an IPv4 datagram carries on a 1500-byte MTU. @@ -237,40 +237,165 @@ impl fmt::Display for MediaFecError { impl std::error::Error for MediaFecError {} // --------------------------------------------------------------------------- -// The code: equal-length shards in, equal-length shards out. +// The code: rs-gf256-v1 exactly as the module docs state it. Equal-length +// shards in, equal-length shards out; every caller has already bounded a block +// to 1..=256 shards of one validated length, so nothing here allocates more +// than the block's shards and three `k x k` matrices. // --------------------------------------------------------------------------- -fn parity_shards(data: &[Vec], parity_count: usize) -> Result>, MediaFecError> { - let code = ReedSolomon::new(data.len(), parity_count) - .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; - let mut parity = vec![vec![0_u8; data[0].len()]; parity_count]; - code.encode_sep(data, &mut parity) - .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; - Ok(parity) +/// `x^8 + x^4 + x^3 + x^2 + 1`, the polynomial the field is reduced by. +const FIELD_POLYNOMIAL: u16 = 0x11D; + +/// `EXP[i]` is `2^i`, written out over two periods so the sum of two logarithms +/// indexes it without reduction; `LOG` is its inverse on the non-zero +/// elements. `2` generates the multiplicative group of GF(2^8) under +/// [`FIELD_POLYNOMIAL`], so the tables cover the field. +const FIELD_TABLES: ([u8; 510], [u8; 256]) = { + let mut exp = [0_u8; 510]; + let mut log = [0_u8; 256]; + let mut power: u16 = 1; + let mut i = 0; + while i < 255 { + exp[i] = power as u8; + exp[i + 255] = power as u8; + log[power as usize] = i as u8; + power <<= 1; + if power & 0x100 != 0 { + power ^= FIELD_POLYNOMIAL; + } + i += 1; + } + (exp, log) +}; + +fn gf_mul(a: u8, b: u8) -> u8 { + let (exp, log) = &FIELD_TABLES; + if a == 0 || b == 0 { + return 0; + } + exp[usize::from(log[usize::from(a)]) + usize::from(log[usize::from(b)])] +} + +/// `1 / a` for non-zero `a`. +fn gf_reciprocal(a: u8) -> u8 { + let (exp, log) = &FIELD_TABLES; + exp[255 - usize::from(log[usize::from(a)])] +} + +/// Row `r` of the Vandermonde matrix `V`: `r^0, r^1, .., r^(k-1)`, the row +/// number read as a field element, with `0^0 = 1`. +fn vandermonde_row(r: usize, k: usize) -> Vec { + let point = r as u8; + let mut row = Vec::with_capacity(k); + let mut power = 1_u8; + for _ in 0..k { + row.push(power); + power = gf_mul(power, point); + } + row +} + +/// The inverse of a square matrix, by Gauss-Jordan elimination. +fn invert(mut matrix: Vec>) -> Vec> { + let n = matrix.len(); + let mut inverse: Vec> = + (0..n).map(|r| (0..n).map(|c| u8::from(r == c)).collect()).collect(); + for col in 0..n { + let pivot = (col..n) + .find(|&r| matrix[r][col] != 0) + .expect("rs-gf256-v1 only inverts matrices whose evaluation points are distinct"); + matrix.swap(col, pivot); + inverse.swap(col, pivot); + let scale = gf_reciprocal(matrix[col][col]); + for c in 0..n { + matrix[col][c] = gf_mul(scale, matrix[col][c]); + inverse[col][c] = gf_mul(scale, inverse[col][c]); + } + let (pivot_row, pivot_inverse) = (matrix[col].clone(), inverse[col].clone()); + for r in (0..n).filter(|&r| r != col) { + let factor = matrix[r][col]; + for c in 0..n { + matrix[r][c] ^= gf_mul(factor, pivot_row[c]); + inverse[r][c] ^= gf_mul(factor, pivot_inverse[c]); + } + } + } + inverse +} + +/// The given rows of the encoding matrix `E = V * inverse(top)` for a block of +/// `k` data shards, where `top` is the first `k` rows of `V`. +fn encoding_rows(k: usize, rows: impl Iterator) -> Vec> { + let top_inverse = invert((0..k).map(|r| vandermonde_row(r, k)).collect()); + rows.map(|r| { + let v = vandermonde_row(r, k); + (0..k) + .map(|c| (0..k).fold(0, |sum, i| sum ^ gf_mul(v[i], top_inverse[i][c]))) + .collect() + }) + .collect() +} + +/// Byte `j` of the result is the sum over `i` of `coefficients[i] * shards[i][j]`. +/// +/// The hot loop of both encode and recovery: each coefficient's 256 products +/// are tabulated once, so a shard byte costs one lookup and one XOR. +fn combine(coefficients: &[u8], shards: &[&[u8]], shard_bytes: usize) -> Vec { + let mut out = vec![0_u8; shard_bytes]; + for (&coefficient, shard) in coefficients.iter().zip(shards) { + match coefficient { + 0 => {} + 1 => { + for (byte, &input) in out.iter_mut().zip(shard.iter()) { + *byte ^= input; + } + } + _ => { + let products: [u8; 256] = std::array::from_fn(|x| gf_mul(coefficient, x as u8)); + for (byte, &input) in out.iter_mut().zip(shard.iter()) { + *byte ^= products[usize::from(input)]; + } + } + } + } + out +} + +/// The `parity_count` parity shards of `data`: rows `k..k + m` of `E`. +fn parity_shards(data: &[Vec], parity_count: usize) -> Vec> { + let k = data.len(); + let inputs: Vec<&[u8]> = data.iter().map(Vec::as_slice).collect(); + encoding_rows(k, k..k + parity_count) + .iter() + .map(|row| combine(row, &inputs, data[0].len())) + .collect() } /// `shards` holds the block's `k` data slots then `m` parity slots, `None` for -/// each missing. Returns the `k` data shards. +/// each missing. Returns the `k` data shards, recovered from the first `k` +/// present shards through the inverse of their rows of `E`. fn recover_data_shards( data_shards: usize, - mut shards: Vec>>, + shards: Vec>>, ) -> Result>, MediaFecError> { - let parity_shards = shards.len() - data_shards; - let present = shards.iter().flatten().count(); - if present < data_shards { + let present: Vec = (0..shards.len()) + .filter(|&slot| shards[slot].is_some()) + .take(data_shards) + .collect(); + if present.len() < data_shards { return Err(MediaFecError::BeyondRepair { - shards_present: present, + shards_present: present.len(), shards_needed: data_shards, }); } - let code = ReedSolomon::new(data_shards, parity_shards) - .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; - code.reconstruct_data(&mut shards) - .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; - Ok(shards - .into_iter() - .take(data_shards) - .map(|shard| shard.expect("reconstruct_data fills every data shard")) + let inputs: Vec<&[u8]> = present.iter().map(|&slot| shards[slot].as_deref().unwrap()).collect(); + let shard_bytes = inputs[0].len(); + let decode = invert(encoding_rows(data_shards, present.iter().copied())); + Ok((0..data_shards) + .map(|slot| match &shards[slot] { + Some(shard) => shard.clone(), + None => combine(&decode[slot], &inputs, shard_bytes), + }) .collect()) } @@ -393,7 +518,7 @@ pub fn protect_video_frame( .map(|chunk| padded(&chunk.payload, shard_bytes)) .collect(); let parity_count = policy.video_parity_shards(size); - let parity = parity_shards(&data, usize::from(parity_count))?; + let parity = parity_shards(&data, usize::from(parity_count)); let parity_records: Vec = parity .into_iter() @@ -634,7 +759,7 @@ pub fn protect_audio_block( let data: Vec> = packets.iter().map(|packet| packet.payload.clone()).collect(); let deadline_ticks = packets.iter().map(|packet| packet.deadline_ticks).max().unwrap_or(0); let parity: Vec = - parity_shards(&data, usize::from(policy.audio_parity_shards))? + parity_shards(&data, usize::from(policy.audio_parity_shards)) .into_iter() .enumerate() .map(|(parity_index, payload)| GameCultMediaAudioParityShardRecord { diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 3ee6af66..f91a3f1e 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -28,6 +28,9 @@ use crate::create_reconnect_policy; use crate::decode_cultnet_message_from_slice; use crate::encode_cultnet_message_to_vec; +#[cfg(test)] +mod send_failure_tests; + const RUDP_MAGIC: [u8; 4] = [0x43, 0x4e, 0x52, 0x30]; const RUDP_VERSION: u8 = 0; pub(crate) const RUDP_FIXED_HEADER_BYTES: usize = 36; @@ -53,6 +56,13 @@ const RUDP_RECEIVE_AHEAD_WINDOW: u32 = 1_024; const RUDP_CONNECT_ATTEMPT_MS: u64 = 3_000; const RUDP_MAX_ORDERED_BUFFERED_FRAMES: usize = 1_024; const RUDP_MAX_ORDERED_BUFFERED_BYTES: usize = 4 * 1024 * 1024; +/// The sender's flow window: a reliable packet goes on the wire only while its +/// sequence is at most this far above the lowest unacked one, and the payload +/// above that sequence stays within the byte bound. A receiver holds +/// `RUDP_RECEIVE_AHEAD_WINDOW - 1` sequences and `RUDP_MAX_ORDERED_BUFFERED_BYTES` +/// beyond its watermark, so a sender that obeys this never overruns it. +const RUDP_FLOW_WINDOW_SEQUENCES: u32 = RUDP_RECEIVE_AHEAD_WINDOW - 1; +const RUDP_FLOW_WINDOW_BYTES: usize = RUDP_MAX_ORDERED_BUFFERED_BYTES; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CultNetRudpPacketType { @@ -244,6 +254,11 @@ pub struct CultNetRudpSession { received_sequences: BTreeSet, latest_sequenced_by_channel: BTreeMap, pending_reliable: BTreeMap, + /// Payload sizes of acknowledged reliable sequences above the lowest + /// unacknowledged one. The receiver still holds those bytes behind the gap, + /// so the flow window keeps counting them until the lowest sequence + /// advances past them. + acked_above_lowest: BTreeMap, queued_reliable: VecDeque<(CultNetRudpPacket, Option)>, reliable_packets_expired: u64, /// Every reliable sequence up to and including this one has been received @@ -284,6 +299,7 @@ impl CultNetRudpSession { received_sequences: BTreeSet::new(), latest_sequenced_by_channel: BTreeMap::new(), pending_reliable: BTreeMap::new(), + acked_above_lowest: BTreeMap::new(), queued_reliable: VecDeque::new(), reliable_packets_expired: 0, received_through: None, @@ -368,6 +384,7 @@ impl CultNetRudpSession { self.awaiting_accept = false; self.generation += 1; self.pending_reliable.clear(); + self.acked_above_lowest.clear(); self.queued_reliable.clear(); } @@ -451,6 +468,28 @@ impl CultNetRudpSession { self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `is_permanent_send_error`). Resending it would fail forever, + /// and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. Returns the goodbye for the peer; + /// its reason names the error. + pub fn end_unsendable_session(&mut self, error: &std::io::Error) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(format!("{RUDP_UNSENDABLE_PACKET_REASON}: {error}").into_bytes()) + } + + /// Undoes a send that never reached the wire: every packet it created at + /// or after `first_sequence`, pending or queued, leaves no trace and its + /// sequences are reissued, so the peer never sees a gap. Only valid while + /// nothing else has been sent since. + fn withdraw_send(&mut self, first_sequence: u32) { + self.pending_reliable + .retain(|sequence, _| *sequence < first_sequence); + self.queued_reliable + .retain(|(packet, _)| packet.sequence < first_sequence); + self.next_sequence = first_sequence; + } + pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { // A session that has had a peer starts a new generation: nothing it // learned from that peer describes the one this Connect reaches, and @@ -560,8 +599,14 @@ impl CultNetRudpSession { payload: Vec, options: CultNetRudpSendOptions, ) -> Result { + // Measure the window as `send_many` will: without the deadline-passed + // sends it is about to reclaim. + if options.reliable { + self.purge_expired_reliable(options.now_ms); + } if options.reliable - && self.pending_reliable.len() >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + && (!self.queued_reliable.is_empty() + || !self.window_admits(self.next_sequence, payload.len())) { return Err(anyhow!( "RUDP reliable send window is full; receive acknowledgements before sending" @@ -1073,22 +1118,47 @@ impl CultNetRudpSession { self.queued_reliable .retain(|(_, expires_at_ms)| expires_at_ms.is_none_or(|at| now_ms <= at)); let after = self.pending_reliable.len() + self.queued_reliable.len(); + if after != before { + self.forget_acked_below_lowest(); + } self.reliable_packets_expired = self .reliable_packets_expired .saturating_add((before - after) as u64); } + /// Whether a reliable packet may go on the wire now: the window has a slot, + /// and its sequence and the payload above the lowest unacked sequence stay + /// inside the flow window. With nothing pending, any packet is admissible. + fn window_admits(&self, sequence: u32, payload_len: usize) -> bool { + if self.pending_reliable.len() >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS { + return false; + } + let Some((&lowest, _)) = self.pending_reliable.first_key_value() else { + return true; + }; + let pending_bytes: usize = self + .pending_reliable + .iter() + .filter(|(pending_sequence, _)| **pending_sequence > lowest) + .map(|(_, pending)| pending.packet.payload.len()) + .sum(); + let acked_bytes: usize = self.acked_above_lowest.values().sum(); + let bytes_above = pending_bytes + acked_bytes; + sequence.saturating_sub(lowest) <= RUDP_FLOW_WINDOW_SEQUENCES + && bytes_above + payload_len <= RUDP_FLOW_WINDOW_BYTES + } + fn admit_reliable_packets( &mut self, packets: Vec, now_ms: u64, expires_at_ms: Option, ) -> Vec { - let available = - CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS.saturating_sub(self.pending_reliable.len()); - let mut ready = Vec::with_capacity(available.min(packets.len())); + let mut ready = Vec::new(); for packet in packets { - if ready.len() < available { + if self.queued_reliable.is_empty() + && self.window_admits(packet.sequence, packet.payload.len()) + { self.track_reliable(packet.clone(), now_ms, expires_at_ms); ready.push(packet); } else { @@ -1099,10 +1169,12 @@ impl CultNetRudpSession { } fn promote_queued_reliable(&mut self, now_ms: u64) -> Vec { - let available = - CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS.saturating_sub(self.pending_reliable.len()); - let mut ready = Vec::with_capacity(available.min(self.queued_reliable.len())); - for _ in 0..available { + let mut ready = Vec::new(); + while self + .queued_reliable + .front() + .is_some_and(|(packet, _)| self.window_admits(packet.sequence, packet.payload.len())) + { let Some((packet, expires_at_ms)) = self.queued_reliable.pop_front() else { break; }; @@ -1133,12 +1205,29 @@ impl CultNetRudpSession { } fn apply_acknowledgements(&mut self, packet: &CultNetRudpPacket) { - self.pending_reliable.remove(&packet.ack); + self.acknowledge(packet.ack); for bit in 0..32 { if (packet.ack_mask & (1_u32 << bit)) != 0 && packet.ack > bit { - self.pending_reliable.remove(&(packet.ack - bit - 1)); + self.acknowledge(packet.ack - bit - 1); } } + self.forget_acked_below_lowest(); + } + + fn acknowledge(&mut self, sequence: u32) { + if let Some(pending) = self.pending_reliable.remove(&sequence) { + self.acked_above_lowest + .insert(sequence, pending.packet.payload.len()); + } + } + + /// Acknowledged sizes stop counting once the lowest unacknowledged + /// sequence passes them, and all of them stop when nothing is unacknowledged. + fn forget_acked_below_lowest(&mut self) { + match self.pending_reliable.first_key_value() { + Some((&lowest, _)) => self.acked_above_lowest.retain(|sequence, _| *sequence > lowest), + None => self.acked_above_lowest.clear(), + } } /// The handshake's one act on the watermark: the peer's Connect or Accept @@ -1557,6 +1646,13 @@ pub struct CultNetRudpServerHub { pending_events: VecDeque, pub profile: CultNetTransportProfile, stats: CultNetTransportStats, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultNetRudpServerHub { @@ -1605,6 +1701,10 @@ impl CultNetRudpServerHub { pending_events: VecDeque::new(), profile, stats: CultNetTransportStats::default(), + #[cfg(test)] + failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -1645,7 +1745,7 @@ impl CultNetRudpServerHub { payload: Vec, ) -> Result<()> { let remote_addr = session.remote_addr; - let packets = { + let (first_sequence, packets) = { let peer = self .peers .get_mut(&remote_addr) @@ -1656,7 +1756,8 @@ impl CultNetRudpServerHub { session.session_generation )); } - peer.session.send_many( + let first_sequence = peer.session.next_sequence; + let packets = peer.session.send_many( channel_id, payload, channel_send_options( @@ -1666,10 +1767,25 @@ impl CultNetRudpServerHub { channel_delivery(&self.profile, channel_id), ), self.max_fragment_bytes, - )? + )?; + (first_sequence, packets) }; - for packet in packets { - self.send_packet(remote_addr, &packet)?; + for (index, packet) in packets.iter().enumerate() { + let Some(error) = self.send_packet(remote_addr, packet)? else { + continue; + }; + // A caller-directed send that can never succeed is the caller's + // error and queues nothing. Only when earlier fragments are already + // on the wire has the peer seen part of it, and then the session + // cannot be kept. + if index == 0 { + if let Some(peer) = self.peers.get_mut(&remote_addr) { + peer.session.withdraw_send(first_sequence); + } + } else { + self.end_unsendable_peer(remote_addr, &error)?; + } + return Err(error.into()); } self.stats.frames_sent += 1; Ok(()) @@ -1700,7 +1816,9 @@ impl CultNetRudpServerHub { .remove(&session.remote_addr) .expect("validated RUDP peer exists"); let packet = peer.session.create_disconnect(reason); - self.send_packet(session.remote_addr, &packet)?; + if let Some(error) = self.send_packet(session.remote_addr, &packet)? { + return Err(error.into()); + } Ok(true) } @@ -1758,7 +1876,7 @@ impl CultNetRudpServerHub { self.end_refusing_peer(remote_addr)?; return Ok(true); }; - self.send_packet(remote_addr, &reply)?; + self.send_in_poll(remote_addr, &reply)?; return Ok(true); } if !self.peers.contains_key(&remote_addr) && self.peers.len() >= self.max_peers { @@ -1787,7 +1905,11 @@ impl CultNetRudpServerHub { self.stats.packets_dropped += 1; return Ok(true); }; - self.send_packet(remote_addr, &accept)?; + if self.send_packet(remote_addr, &accept)?.is_some() { + // The peer cannot be answered, so no session starts. + self.stats.packets_dropped += 1; + return Ok(true); + } let replaced = self.peers.insert( remote_addr, CultNetRudpServerPeer { @@ -1821,14 +1943,17 @@ impl CultNetRudpServerHub { } else { None }; - if let Some(reply) = result.reply { - self.send_packet(remote_addr, &reply)?; - } - for ready in result.ready_to_send { - self.send_packet(remote_addr, &ready)?; - } - if let Some(ack) = ack { - self.send_packet(remote_addr, &ack)?; + let mut unsendable = None; + for packet in result + .reply + .iter() + .chain(&result.ready_to_send) + .chain(ack.iter()) + { + unsendable = self.send_packet(remote_addr, packet)?; + if unsendable.is_some() { + break; + } } if result.pong { self.pending_events.push_back(CultNetRudpServerEvent::Pong { @@ -1854,6 +1979,8 @@ impl CultNetRudpServerHub { reason: result.disconnect_reason, }); self.peers.remove(&remote_addr); + } else if let Some(error) = unsendable { + self.end_unsendable_peer(remote_addr, &error)?; } Ok(true) } @@ -1873,7 +2000,37 @@ impl CultNetRudpServerHub { session: peer.context, reason: goodbye.payload.clone(), }); - self.send_packet(remote_addr, &goodbye) + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// An admitted peer owes a packet that can never be sent as built. Inside a + /// poll that ends the peer's session, never the poll: the peer is told, and + /// the caller sees `Disconnected` with a reason that names the error. + fn end_unsendable_peer( + &mut self, + remote_addr: SocketAddr, + error: &std::io::Error, + ) -> Result<()> { + let Some(mut peer) = self.peers.remove(&remote_addr) else { + return Ok(()); + }; + let goodbye = peer.session.end_unsendable_session(error); + self.pending_events + .push_back(CultNetRudpServerEvent::Disconnected { + session: peer.context, + reason: goodbye.payload.clone(), + }); + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// A send inside a poll: a permanent failure ends that peer's session. + fn send_in_poll(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + if let Some(error) = self.send_packet(remote_addr, packet)? { + self.end_unsendable_peer(remote_addr, &error)?; + } + Ok(()) } pub fn poll_resends(&mut self) -> Result<()> { @@ -1888,7 +2045,10 @@ impl CultNetRudpServerHub { ); } for (remote_addr, packet) in packets { - self.send_packet(remote_addr, &packet)?; + // A peer ended earlier in this loop has nothing left to resend. + if self.peers.contains_key(&remote_addr) { + self.send_in_poll(remote_addr, &packet)?; + } } Ok(()) } @@ -1913,11 +2073,44 @@ impl CultNetRudpServerHub { timed_out.into_iter().map(|(_, context)| context).collect() } - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A reliable packet stays pending and is resent; an unreliable one + /// was allowed to be lost. A permanent failure (`is_permanent_send_error`) + /// is returned as `Some`: the datagram can never be sent as built, and the + /// caller decides between refusing the send and ending the peer's session. + /// Only an encode failure, the hub's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; - let sent = self.socket.send_to(&wire, remote_addr)?; - self.stats.bytes_sent += sent as u64; - Ok(()) + match self.send_datagram(&wire, remote_addr) { + Ok(sent) => self.stats.bytes_sent += sent as u64, + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), + Err(_) => self.stats.send_failures += 1, + } + Ok(None) + } + + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + { + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + } + self.socket.send_to(wire, remote_addr) } } @@ -2405,6 +2598,50 @@ where /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; +/// Prefix of the goodbye reason for a session that owed a packet which can +/// never be sent; the error's own description follows it. +const RUDP_UNSENDABLE_PACKET_REASON: &str = "packet could not be sent"; + +/// Whether a failed `send_to` can never succeed for the datagram as built, so +/// resending it is pointless: it is too large for a datagram (`EMSGSIZE`), its +/// address is malformed for the socket (`EINVAL`), or belongs to another +/// address family (`EAFNOSUPPORT`). Everything else (no route, full buffers, +/// firewall drops, a refusal reported for an earlier datagram, an interrupted +/// call) is a property of the path or the moment and may pass, so it is a lost +/// datagram, not a reason to end a session. +pub fn is_permanent_send_error(error: &std::io::Error) -> bool { + // `EMSGSIZE` and `EAFNOSUPPORT`, or their Winsock equivalents. + #[cfg(any(target_os = "linux", target_os = "android"))] + const PERMANENT_OS_ERRORS: [i32; 2] = [90, 97]; + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + const PERMANENT_OS_ERRORS: [i32; 2] = [40, 47]; + #[cfg(windows)] + const PERMANENT_OS_ERRORS: [i32; 2] = [10040, 10047]; + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", + windows + )))] + const PERMANENT_OS_ERRORS: [i32; 0] = []; + // `InvalidInput` is `EINVAL` and `WSAEINVAL`. + error.kind() == ErrorKind::InvalidInput + || error + .raw_os_error() + .is_some_and(|code| PERMANENT_OS_ERRORS.contains(&code)) +} /// A session cannot admit a Connect when its sequence space starts exhausted or /// its reliable queue holds nothing, so neither is a usable configuration. diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs new file mode 100644 index 00000000..4bff8cf9 --- /dev/null +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -0,0 +1,502 @@ +//! One peer whose datagrams cannot be sent must not stop the hub serving the +//! others. The failure is injected at `send_datagram` through `failing_peers`. + +use super::*; +use std::time::Duration; + +const CONNECTION_ID: u32 = 0x4355_4c54; + +fn socket() -> UdpSocket { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + socket +} + +fn client(remote: SocketAddr) -> CultNetRudpSocketTransportConnection { + let mut options = + CultNetRudpSocketTransportOptions::client("peer", socket(), remote, CONNECTION_ID); + options.resend_delay_ms = 10; + CultNetRudpSocketTransportConnection::new(options).unwrap() +} + +fn connect( + hub: &mut CultNetRudpServerHub, + client: &mut CultNetRudpSocketTransportConnection, +) -> CultNetRudpServerSessionContext { + client.connect(Vec::new()).unwrap(); + let mut session = None; + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Connected { session: found }) = + hub.receive_event_once().unwrap() + { + session = Some(found); + break; + } + } + client.receive_once().unwrap(); + assert!(client.connected()); + session.expect("hub saw the Connect") +} + +/// Two connected peers, ordered so the first is the one the hub visits first +/// (the peer map is ordered by address). +fn two_peers() -> ( + CultNetRudpServerHub, + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut a = client(server_addr); + let mut b = client(server_addr); + let a_session = connect(&mut hub, &mut a); + let b_session = connect(&mut hub, &mut b); + if a_session.remote_addr < b_session.remote_addr { + (hub, (a, a_session), (b, b_session)) + } else { + (hub, (b, b_session), (a, a_session)) + } +} + +#[test] +fn a_failed_resend_to_one_peer_does_not_stop_the_others_resends() { + let (mut hub, (_x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + // Y's path recovers; X's stays dead. X is visited first. + hub.failing_peers.remove(&y_session.remote_addr); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + assert!(hub.stats().send_failures >= 3); + assert_eq!(hub.sessions().len(), 2, "a send failure ends no session"); +} + +#[test] +fn a_failed_reply_to_one_peer_does_not_stop_the_hub_serving_the_others() { + let (mut hub, (mut x, x_session), (mut y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + + x.send("schema", b"from-x".to_vec()).unwrap(); + y.send("schema", b"from-y".to_vec()).unwrap(); + + let mut frames = Vec::new(); + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Frame { frame, .. }) = hub.receive_event_once().unwrap() + { + frames.push(frame.payload); + } + if frames.len() == 2 { + break; + } + } + frames.sort(); + assert_eq!(frames, vec![b"from-x".to_vec(), b"from-y".to_vec()]); + assert!( + hub.stats().send_failures >= 1, + "X's ack was counted as lost" + ); + assert_eq!(hub.sessions().len(), 2); +} + +// A permanent send failure is one that can never succeed for the datagram as +// built. A caller-directed send returns it and queues nothing; inside a poll it +// ends that peer's session and never the poll. + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +fn next_event(hub: &mut CultNetRudpServerHub) -> CultNetRudpServerEvent { + for _ in 0..20 { + if let Some(event) = hub.receive_event_once().unwrap() { + return event; + } + } + panic!("the hub raised no event"); +} + +fn one_peer( + max_fragment_bytes: Option, +) -> ( + CultNetRudpServerHub, + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + options.max_fragment_bytes = max_fragment_bytes; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + let session = connect(&mut hub, &mut peer); + (hub, peer, session) +} + +#[test] +fn an_oversized_send_returns_the_error_and_queues_nothing() { + let (mut hub, mut peer, session) = one_peer(None); + // The hub's own Accept is still awaiting its ack; the refused send adds nothing to it. + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); + + let error = hub + .send(&session, "schema", vec![7; 70_000]) + .expect_err("70,000 bytes cannot be one datagram"); + let io_error = error + .downcast_ref::() + .expect("the send error is the socket's own"); + assert!(is_permanent_send_error(io_error), "{io_error}"); + assert_eq!(hub.stats().send_failures, 0); + assert_eq!(hub.stats().frames_sent, 0); + assert_eq!(hub.sessions().len(), 1, "the caller's error ends no session"); + let remote = session.remote_addr; + assert_eq!( + hub.peers[&remote].session.outstanding_reliable_packet_count(), + outstanding, + "nothing is pending or queued to resend" + ); + + // The refused send consumed no sequence: the next frame is delivered in + // order instead of waiting behind a gap that never fills. + hub.send(&session, "schema", b"next".to_vec()).unwrap(); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = peer.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("the next frame arrives").payload, b"next"); +} + +#[test] +fn a_send_that_fails_permanently_after_a_fragment_left_ends_the_session() { + let (mut hub, _peer, session) = one_peer(Some(1000)); + hub.unsendable_after.insert(session.remote_addr, 1); + + let error = hub + .send(&session, "schema", vec![7; 3000]) + .expect_err("the second fragment can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); + assert!(hub.sessions().is_empty()); + let CultNetRudpServerEvent::Disconnected { reason, .. } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert!(is_unsendable_reason(&reason)); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut hub, (mut x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + + // X's datagram can now never be sent; Y's path recovers. X is visited first. + hub.failing_peers.clear(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + assert_eq!(hub.sessions(), vec![y_session.clone()]); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + // The goodbye reaches X and names the error. + for _ in 0..20 { + x.receive_once().unwrap(); + if x.disconnect_reason().is_some() { + break; + } + } + assert!(is_unsendable_reason( + x.disconnect_reason().expect("X is told its session ended") + )); +} + +#[test] +fn a_permanent_failure_replying_to_a_frame_ends_the_session_after_the_frame_is_delivered() { + let (mut hub, (mut x, x_session), (_y, _y_session)) = two_peers(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + x.send("schema", b"from-x".to_vec()).unwrap(); + + let CultNetRudpServerEvent::Frame { frame, .. } = next_event(&mut hub) else { + panic!("the frame is delivered before the session ends"); + }; + assert_eq!(frame.payload, b"from-x"); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the session ends"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + assert_eq!(hub.sessions().len(), 1); +} + +#[test] +fn a_permanent_failure_answering_a_repeated_connect_ends_the_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let raw = socket(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(5), + resend_delay_ms: 10, + max_pending_reliable_packets: None, + }); + let wire = encode_rudp_packet(&session.create_connect(0, Vec::new()).unwrap()).unwrap(); + + raw.send_to(&wire, server_addr).unwrap(); + let CultNetRudpServerEvent::Connected { session: admitted } = next_event(&mut hub) else { + panic!("the first Connect is admitted"); + }; + hub.unsendable_after.insert(raw.local_addr().unwrap(), 0); + raw.send_to(&wire, server_addr).unwrap(); + + let CultNetRudpServerEvent::Disconnected { + session: ended, + reason, + } = next_event(&mut hub) + else { + panic!("the reply to the repeat can never be sent, so the session ends"); + }; + assert_eq!(ended, admitted); + assert!(is_unsendable_reason(&reason)); + assert!(hub.sessions().is_empty()); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + hub.unsendable_after + .insert(peer.socket.local_addr().unwrap(), 0); + + peer.connect(Vec::new()).unwrap(); + for _ in 0..5 { + assert_eq!(hub.receive_event_once().unwrap(), None); + } + assert!(hub.sessions().is_empty()); + assert_eq!(hub.stats().packets_dropped, 1); +} + +#[test] +fn a_healthy_send_counts_its_bytes_and_no_failure() { + let (mut hub, mut peer, session) = one_peer(None); + let before = hub.stats().bytes_sent; + let received_before = peer.stats().bytes_received; + hub.send(&session, "schema", b"hello".to_vec()).unwrap(); + for _ in 0..20 { + if peer.receive_once().unwrap().is_some() { + break; + } + } + let sent = hub.stats().bytes_sent - before; + assert!(sent > b"hello".len() as u64); + assert_eq!(sent, peer.stats().bytes_received - received_before); + assert_eq!(hub.stats().send_failures, 0); +} + +#[test] +fn only_a_datagram_that_can_never_be_sent_is_a_permanent_failure() { + use std::io::{Error, ErrorKind}; + assert!(is_permanent_send_error(&Error::from(ErrorKind::InvalidInput))); + assert!(!is_permanent_send_error(&Error::other("no os code"))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::WouldBlock))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::ConnectionReset))); + #[cfg(target_os = "linux")] + let (permanent, transient) = ( + // EMSGSIZE, EAFNOSUPPORT, EINVAL + [90, 97, 22], + // EPERM, EINTR, EAGAIN, EACCES, ENETDOWN, ENETUNREACH, EADDRNOTAVAIL, ENOBUFS, ECONNREFUSED, EHOSTUNREACH + [1, 4, 11, 13, 100, 101, 99, 105, 111, 113], + ); + #[cfg(windows)] + let (permanent, transient) = ( + // WSAEMSGSIZE, WSAEAFNOSUPPORT, WSAEINVAL + [10040, 10047, 10022], + // WSAEACCES, WSAEWOULDBLOCK, WSAENETDOWN, WSAENETUNREACH, WSAEADDRNOTAVAIL, WSAENOBUFS, WSAECONNRESET, + // WSAECONNREFUSED, WSAEHOSTUNREACH, WSAEINTR + [10013, 10035, 10050, 10051, 10049, 10055, 10054, 10061, 10065, 10004], + ); + #[cfg(any(target_os = "linux", windows))] + { + for code in permanent { + assert!( + is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is permanent" + ); + } + for code in transient { + assert!( + !is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is transient" + ); + } + } +} + +#[test] +fn a_permanent_failure_sending_a_disconnect_is_returned_to_the_caller() { + let (mut hub, _peer, session) = one_peer(None); + hub.unsendable_after.insert(session.remote_addr, 0); + + let error = hub + .disconnect(&session, b"bye".to_vec()) + .expect_err("the goodbye can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); +} + +#[test] +fn a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut hub, (_x, x_session), (_y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + hub.send(&x_session, "schema", b"one".to_vec()).unwrap(); + hub.send(&x_session, "schema", b"two".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + // Both resends are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(hub.stats().send_failures, 3); + assert_eq!(hub.sessions().len(), 1); +} + +#[test] +fn a_refused_send_withdraws_the_fragments_the_window_queued_as_well() { + let (mut hub, _peer, session) = one_peer(Some(1)); + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); + hub.unsendable_after.insert(session.remote_addr, 0); + + // 1,100 one-byte fragments: the flow window admits the first ones and + // queues the rest, and the first datagram can never be sent. + hub.send(&session, "schema", vec![7; 1_100]) + .expect_err("the first fragment can never be sent"); + + assert_eq!( + hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(), + outstanding, + "nothing of the refused send stays queued" + ); +} + +fn receive_frame( + peer: &mut CultNetRudpSocketTransportConnection, + hub: &mut CultNetRudpServerHub, +) -> Option> { + for _ in 0..600 { + if let Some(frame) = peer.receive_once().unwrap() { + return Some(frame.payload); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + None +} + +/// Wherever in a fragmented send the datagram that can never be sent falls, the +/// peer never delivers a frame built from it. At the first fragment nothing left, +/// the send is withdrawn, and the next send is delivered whole; later, the peer has +/// seen part of it, and the session ends with nothing delivered. +#[test] +fn a_send_refused_at_any_fragment_delivers_nothing_of_it() { + for fragments in [5_usize, 40] { + for failed in 0..fragments { + let (mut hub, mut peer, session) = one_peer(Some(1000)); + hub.send(&session, "schema", b"warm".to_vec()).unwrap(); + assert_eq!(receive_frame(&mut peer, &mut hub).as_deref(), Some(&b"warm"[..])); + let refused: Vec = (0..fragments * 1000).map(|i| (i % 251) as u8).collect(); + hub.unsendable_after.insert(session.remote_addr, failed); + let result = hub.send(&session, "schema", refused); + if hub.unsendable_after.remove(&session.remote_addr).is_some() { + // The flow window queued that fragment rather than sending it now. + assert!(result.is_ok(), "{fragments}/{failed}: not reached, yet refused"); + continue; + } + assert!(result.is_err(), "{fragments}/{failed}"); + if failed == 0 { + assert_eq!(hub.sessions().len(), 1, "{fragments}/{failed}"); + let next: Vec = (0..fragments * 1000).map(|i| ((i % 13) + 100) as u8).collect(); + hub.send(&session, "schema", next.clone()).unwrap(); + let delivered = receive_frame(&mut peer, &mut hub).expect("the next send arrives"); + assert!(delivered == next, "{fragments}/{failed}: delivered something else"); + for _ in 0..30 { + if let Some(frame) = peer.receive_once().unwrap() { + panic!("{fragments}/{failed}: a stray frame of {} bytes", frame.payload.len()); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + } else { + assert!(hub.sessions().is_empty(), "{fragments}/{failed}"); + for _ in 0..30 { + match peer.receive_once() { + Ok(Some(frame)) => panic!( + "{fragments}/{failed}: the peer delivered {} bytes", + frame.payload.len() + ), + Ok(None) => {} + Err(_) => break, + } + } + } + } + } +} diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index 71e714c4..f2c196d4 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -31,6 +31,11 @@ pub struct CultNetTransportStats { /// unadmitted sender, or a packet the session refused. A steady rate means /// a flow is being moved or scanned, not that this transport is failing. pub packets_dropped: u64, + /// Datagrams a multi-peer server could not send to one peer. Each is a lost + /// datagram attributed to that peer: reliable packets stay pending and are + /// resent, and no other peer's poll is affected. Zero for single-peer + /// transports, whose callers get the send error itself. + pub send_failures: u64, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index 46e6b2e2..2f4572d9 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -1334,6 +1334,71 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { Ok(()) } +fn rudp_client_bound_host(endpoint: &str, bind_host: Option<&str>) -> Result { + let client = CultMesh::create_rudp_client_for_endpoint( + "rust-cultmesh-bind-client", + 0x2030_4060, + endpoint, + CultMeshRudpSocketOptions { + bind_host: bind_host.map(str::to_string), + ..CultMeshRudpSocketOptions::default() + }, + )?; + Ok(client.profile.transports[0] + .host + .clone() + .expect("RUDP client profile advertises its bound host")) +} + +/// A client bound to loopback cannot send to another host on Windows, so a +/// client for a remote endpoint binds the unspecified address of its family. +#[test] +fn cultmesh_rudp_client_for_a_remote_endpoint_binds_the_unspecified_address() -> Result<()> { + assert_eq!(rudp_client_bound_host("rudp://10.77.0.1:17872", None)?, "0.0.0.0"); + assert_eq!(rudp_client_bound_host("rudp://[2001:db8::1]:17872", None)?, "::"); + Ok(()) +} + +#[test] +fn cultmesh_rudp_client_for_a_loopback_endpoint_binds_loopback() -> Result<()> { + assert_eq!(rudp_client_bound_host("rudp://127.0.0.1:17872", None)?, "127.0.0.1"); + assert_eq!(rudp_client_bound_host("rudp://[::1]:17872", None)?, "::1"); + Ok(()) +} + +#[test] +fn cultmesh_rudp_client_binds_an_explicit_bind_host_whatever_the_endpoint() -> Result<()> { + assert_eq!( + rudp_client_bound_host("rudp://10.77.0.1:17872", Some("127.0.0.1"))?, + "127.0.0.1" + ); + assert_eq!( + rudp_client_bound_host("rudp://127.0.0.1:17872", Some("0.0.0.0"))?, + "0.0.0.0" + ); + Ok(()) +} + +#[test] +fn cultmesh_rudp_server_binds_loopback_unless_told_otherwise() -> Result<()> { + let server = CultMesh::create_rudp_server( + "rust-cultmesh-bind-server", + 0x2030_4061, + CultMeshRudpSocketOptions::default(), + )?; + assert_eq!(server.profile.transports[0].host.as_deref(), Some("127.0.0.1")); + let server = CultMesh::create_rudp_server( + "rust-cultmesh-bind-server", + 0x2030_4062, + CultMeshRudpSocketOptions { + bind_host: Some("0.0.0.0".to_string()), + ..CultMeshRudpSocketOptions::default() + }, + )?; + assert_eq!(server.profile.transports[0].host.as_deref(), Some("0.0.0.0")); + Ok(()) +} + #[test] fn cultmesh_facade_creates_rudp_client_from_peer_endpoint() -> Result<()> { let connection_id = 0x2030_4050; diff --git a/packages/cultnet-rs/tests/fixtures/media_fec_rs_gf256_v1.kat b/packages/cultnet-rs/tests/fixtures/media_fec_rs_gf256_v1.kat new file mode 100644 index 00000000..4df4617f --- /dev/null +++ b/packages/cultnet-rs/tests/fixtures/media_fec_rs_gf256_v1.kat @@ -0,0 +1,1322 @@ +# rs-gf256-v1 known answers for the CultNet media FEC codec. +# +# Produced by reed-solomon-erasure 6.0.0 (galois_8; encode_sep, then +# reconstruct_data, which returned the original data for every pattern below), +# through generate_rs_gf256_v1_known_answers in tests/media_fec.rs at CultLib +# commit 73bf4421, before cultnet-rs replaced the crate with its own +# implementation of the construction documented in src/media_fec.rs. +# If these bytes ever change, rs-gf256-v1 changed meaning: bump the scheme id. +# +# case a block of k data and m parity shards +# fill lcg: data shard i is bytes(k * 65536 + m * 256 + i, shard_bytes), where +# bytes(seed, n) starts state = seed * 2654435761 + 1 and n times sets +# state = state * 1664525 + 1013904223 and emits state >> 24, all +# arithmetic on unsigned 32-bit integers, wrapping. +# fill zero: every data byte is 0x00. fill ff: every data byte is 0xff. +# parity

parity shard p, lowercase hex +# erase a pattern of erased shards (0..k data, k..k+m parity); the +# shards left recover the erased data exactly +case 1 2 37 lcg +parity 0 1d76461ffe37cefddcf40fda2b204102be510c9f0d5207339f2a8a89c6c91c7d1c2e1e0c7f +parity 1 1d76461ffe37cefddcf40fda2b204102be510c9f0d5207339f2a8a89c6c91c7d1c2e1e0c7f +erase 0 +erase 0,1 +case 2 2 37 lcg +parity 0 dd18e10df0fc6c0fa9d8e5e86e4fd21a766207cfd40f5b1f846d30267007b52a7d947ab4f5 +parity 1 db201de79683235efa2a5e1150c2cb417ed32b0a58354c1dd1e80b93460ffa8200ec6c14c2 +erase 0,1 +erase 0,2 +erase 1,2 +case 3 2 37 lcg +parity 0 9f31dd8658475a0b0efce438c7f7591a0596afdc19fe90fce15355887c7de3ca42c39466a9 +parity 1 8db62234c2ba4073f6367b9738b0fb00ad4b5908c5c5482725cbbc1b83452c75f9dc380860 +erase 0,1 +erase 0,3 +erase 1,2 +erase 1,3 +case 4 2 37 lcg +parity 0 a8ee004fd095063619e7584588e5a31cc94e9820e3a5d2d55c36b1e570ce9f68556bf67028 +parity 1 98d97eef9511077e81e7a54752088dd5c122b9c9f2ab3dd78e98f1435ec6709f8f49edd0a3 +erase 0,1 +erase 0,4 +erase 2,3 +erase 2,4 +case 5 2 37 lcg +parity 0 4cbfa04fe94c62ca970a747f837544ca91e95d8364637dcb67f43eb3f829f26edd276081e3 +parity 1 d8be8a2b22f6180407e961be1d6804df6eb56cd85e0e78d4a56b54aa4832a316a9c71fe1d1 +erase 0,1 +erase 0,5 +erase 2,5 +erase 3,4 +case 6 2 37 lcg +parity 0 d61fc5398802d8e9d450d52e1c03f37233b7c586fad4ea84a5a81f6f535ee5fbef8f12a068 +parity 1 ca4789f2758e3a32dd0131195eb49d293b2279d6830a0c9a5c39685cbd592a937042dac379 +erase 0,1 +erase 0,6 +erase 3,6 +erase 4,5 +case 7 2 37 lcg +parity 0 63f9bbbf4551c9c0155b03f6ca672253a3884cad31a3609a5293e5570d96039213b9b7d898 +parity 1 d833010b405b5ed00222ce40c128900dd93f0ecc8d9f6ee6e45000653463d7c3ae0d2deeae +erase 0,1 +erase 0,7 +erase 3,7 +erase 5,6 +case 8 2 37 lcg +parity 0 1f2644cdeef6330582964313ac2a88779168a371ce25e9a331a2feb0e926e80a3fb6dc956e +parity 1 07c99275ea440ffe50a30e803a3cc7b399ab77b35c1e394fd5830164d43b8863cc1128849b +erase 0,1 +erase 0,8 +erase 4,8 +erase 6,7 +case 9 3 37 lcg +parity 0 1783c9967a6b04b7747a8311d8d20945a84c966d9de43a32e8ab7542ae536455acee379d42 +parity 1 7daef83cb52401f06d8e470a46bbcadbafc1424629376b5a9125ef2c0587ffe9d84bad60c4 +parity 2 1faeacfdc3f9e3784c134f037fd1e21646d9fd81d0028752a00a4aae093797896fe4c700e6 +erase 0,1,2 +erase 0,4,9 +erase 4,9,10 +erase 6,7,8 +case 10 3 37 lcg +parity 0 62af6d8e84589baa9d526110950638f093f96b94a45b2d9c30b6a9745234574b9ead9e066f +parity 1 dc28ca8f0e824db76227dbaf40733617dbe74da368e0538121c11a60e82bc4e3b47c8fb74e +parity 2 792cad82f9efcf8246d704887e1929eb262662d402ad1a1105d191b8e8fa6511f901000350 +erase 0,1,2 +erase 0,5,10 +erase 5,10,11 +erase 7,8,9 +case 11 3 37 lcg +parity 0 54ff5670647c20b41f8ceb38e74267f6aaa30804c845d4900e44db8ca11c0f3f0751ff5eed +parity 1 8e3025f78568c3c891d514c143e94db1c9886f45c82af0229a8a1222f14ebf6e761a497780 +parity 2 5a35595594b8eafac3a78f9cd182c985ab079a03fd844c7bc1bc473add2e24b9e6aabb6107 +erase 0,1,2 +erase 0,5,11 +erase 5,11,12 +erase 8,9,10 +case 12 3 37 lcg +parity 0 46e7b0691080090c58b9665a5f1c02c559e279cbb817e8cef8d1a2614e96686fbdfb91a03f +parity 1 70ce7f47fe7e88d78c4e0e8958e93c8b0e5543a62d7c61d55959031e518333fb131342470a +parity 2 925e128717e460de9d9c3e5f1d8a0a98e56c203e74a8dc831262bfd648b79ba2566c80ea79 +erase 0,1,2 +erase 0,6,12 +erase 6,12,13 +erase 9,10,11 +case 13 4 37 lcg +parity 0 5dba6319228d9ffa4d4b314912b2f6463a8c1ff416a1307299b9a0994ee8e85ab12a8cf659 +parity 1 a473d1a714554131a8b900d9359fa176c2c2562f3ce23341c592e3fefa6fd3db019d8e77e8 +parity 2 485ac4a9c57938083530f17f2133f8bacadeb7b8f51540daf9266fc6a8a9b95dc781d58cb0 +parity 3 f2de139b1450bb18d36906a9883d94f5fcd1d50574654f467df2ebd5e7057f7e0edb001fc2 +erase 0,1,2,3 +erase 0,6,13,14 +erase 6,13,14,15 +erase 9,10,11,12 +case 14 4 37 lcg +parity 0 d6fc31195735a1ddf12dc46e1769d12101aa30c0775e05ebe21fcf6a741e3e80f0a2222eb6 +parity 1 f41477b7e559afc4bc64ee9094ea1582996327f11dc7f9c07e95456981810d8966810f8ee7 +parity 2 f48caa91c33fd3601dbf9b8d4c650811811b4ccee2bb874125319d8fee4352229ef5ae988b +parity 3 19a11e9ce28d3af19a6a896c3d59941aed6c97cbbf851bca8e710b4bb9354c667aa7e6a8be +erase 0,1,2,3 +erase 0,7,14,15 +erase 7,14,15,16 +erase 10,11,12,13 +case 15 4 37 lcg +parity 0 436fd02724e4f85321b35365f4e8fbf0b8e59172b59a05c0b133fee9efc201c802249660ef +parity 1 5bcc7fa05ccdcfe4056842ff288d85716bc68b61fdc4d00e60c1c828a1737c1e902fdef318 +parity 2 e426dba0f19e3460737f46901181a5bb36cd6e5428a8cb8e54c8e8869838eaa0f136ea659d +parity 3 7c3f1eb480e565aad5df965694b15d758e010f5f85df523585762922c16bd74569c9921b4c +erase 0,1,2,3 +erase 0,7,15,16 +erase 7,15,16,17 +erase 11,12,13,14 +case 16 4 37 lcg +parity 0 85f104a041c87405e95cc483c09d776675d5820ec5b2f06cb467f6603a6f6c28f82bb7e123 +parity 1 25c276c043503575a53501e07972c6fa4f61faad0602b1c106c825f32094d08a469bf5cc58 +parity 2 c1def77fb8d48ef70aa21ffdc7a53c19535c94f1d9d2a4f9418ba26038abbfb4ea681f4b9b +parity 3 cecdb160ca6d4f767f38aa79601a9b78123c1f6c57a032264146ff7ec479dd12fa9c05cf69 +erase 0,1,2,3 +erase 0,8,16,17 +erase 8,16,17,18 +erase 12,13,14,15 +case 4 2 1 lcg +parity 0 a8 +parity 1 98 +erase 0,1 +erase 0,4 +erase 2,3 +erase 2,4 +case 4 2 37 lcg +parity 0 a8ee004fd095063619e7584588e5a31cc94e9820e3a5d2d55c36b1e570ce9f68556bf67028 +parity 1 98d97eef9511077e81e7a54752088dd5c122b9c9f2ab3dd78e98f1435ec6709f8f49edd0a3 +erase 0,1 +erase 0,4 +erase 2,3 +erase 2,4 +case 4 2 1300 lcg +parity 0 a8ee004fd095063619e7584588e5a31cc94e9820e3a5d2d55c36b1e570ce9f68556bf670284a2589488913ff2c91729b64c6857dacddf1709c4c999d687804effc454640dc56ae2392086c72e5283b9fac105113741de8cca8453864789c22df919591a512eb999af9d6845cf933f15714ec42b35d62d2f57fd9de9edefaa91114ff1ff4abffb647030c6057d9c4c9106a75f6c4ab1063b7ba0d1501c85c1ca6abb77b7cb0cc55cfb5efb037017afeaa3baff4cb0ebfb3469dcd5c757c4ca73201f09d7c7dc660686f0d211188403ae0bdbfdf71da0013d3cd2e4c2f197ff34868eecad4e56c54f14752ab3088aadb26c696b8675ba561e4013abba81fccc430b510c45a13e0488891877cdd5fc9de193a534377b4eccd66b97f3368f8b5e1bf63e2518331940770718c1740e08c2a75f605973194057647d27263467d3f31ab4c7ebf3d7b9f02abff4a2c1d7a83896ff936b2c25cd7fd7b58b3967576e65cae819c71591f64df1d217dbbc866bf7dd7d10efc4f94eecf7b8cd0a8c6f27a9fcf32d97940f5d2accc653c883fa40d4d636e59531cd006766b489b893290a934ab7c1e2c8c61cfd0c218359d97d718b6ff90f5a2d86d6e1de3477f20f92f7893a80c7a5dd8e8f81e43e6f01734dce33676794d043257a63998fd92a4a6fa058f538e890a0e540f81b04bdc9449055427ed6b38da5f98b3a41f4996d796b2b9e0de32cfcfa9eee9ddacbc6bf714f446eb93e324b309c2375cb2e2c3290fd8e6d5ef1ed959fe35febb26c4d44edd13c568b1b827dd9a6e74fbd20061824b2647e612e3eb6eb2e73093b1f12c02fb3a0e03eea185d55b8f0f7657d73d209a99ca7da9fc52014a0278a01bf06fc7dbe3fa501fb1bddfdaeaec1284cb5f1d816160b677db1d69a3cd681680b459051f039417590660eaee8f551865928a50d6e18cc949bff156541a87834c396832a7d9df6dc92425c1b6184e41dffbfbb863826c8dcca2c45019a8f37b5e44e1b3920a177f224e32dc8f4038b37a05de36e8c2d1ceb2ea1efee16954b175b972d27adf19b4cf91a76595ebc7fa36fe4bd913e51c8787bdef2572273d21d4e3d65060a1bcf06a6cbf488c97d1e4e95ad4090be1ee99fdc03cb1055e562456a28025945f7b21ed1c058d2c754af83724da667313366877fe4e75f4be28716e7383db3e02854908fb5117bc94cd2dcaa6162212f5068aa314fe279932d9020be20120052467b75002abfc9552c71042ab5c276dd9fcc0fd5d6e69a3cbd1e23c2c802e7cd8c1ebfc085b4739eb0cb198f734f758d94f19c20c4bc59bf8995caf13a768cac10b9f8de6b0fa005b5c963735edcedc3da07fea83499bf65b6e375682aa7aa9dbf10360d76ff134560ace09f8460060fff0391de5348922653dc19891619720fab3dd46e4ef470b27e3aae767d516d100c5b315a308f3835cee0e6e8d96847ac29db8e3fa06d6bda07d37c6f9759c07cd4a04a72bdbe0ce25ab269778a8c8f2c08aec1401eb23d9df2474dbd53544e10667414f39339c154033a4c0ca44a6ab631c67ffd1ac1e52b42f03d6b2fecb4bd22a4c4e25c0722ed5111039ee32e55d3c5b84478851065b9b7c99034e90119a64b4d3c0992ab83a814bc661f6ea13b4694783857903eb31d3739858a729e652df22000ae17c6723af64607ef7323392d74e4bc647058d38b436aa3573149a296dbc9dab14ae415585822e6eaf63fe7f1e6a22b2280204d9a0bebd0bf68f80a057c42c81b48ca28494569b407d1b01c05b078a23bceba8aed30ccf6e192dfc53be1e6890a104261ceb3d13a5d4b1057dbaa8c2554f294ad1 +parity 1 98d97eef9511077e81e7a54752088dd5c122b9c9f2ab3dd78e98f1435ec6709f8f49edd0a3fa193eb71a66f7374036d453d4f639bcc7fb5f6c8b4b6932ee123eb6fbaec4bcdab7bd7b914beff31eb13dcf1c81d5553567b5fdbc838d38216762d8da05136081c250998a2f7abdf12adcf9b4268d58907113293eb1bf8d8f8f0c05833d7cf39cfd20129b951528c31849539c4607e88fb4c08b9c7b432e0ed6e3aa66742f96b9f9e0646f497380bae9ea946f87285a4c372b470dd3bbb8a52771461d42bbf9c5220e301988ac9f0bf944646977627486dd0fd66dbe1a2b63fe0a02e273c17874870e804d2fb2a2faef323058032d11ab7fc7a30f1f783da2c6f3e115e339c35e0997c268dd30aa8bb15705281537c7e85ce8022af05b31602eeadfa8135246ed91129a8901c21729f643f1f7814de50b7cc494ab1a9cb3bf73023d73bb85673fe5fc9a56a75e6fe9c31d53eaf7e175baa118eddf5446f7c0cf6ffd75f616b3d20f1f12b99ab0b78001b4b09fddc1ebda719858008b2991fd268bb1f18c24ea014025b47c1470a51b111d4e5aed12bc5de6e6e684d19d3d5d62d8f71bf61718654acad324f16b1128627fe6de15107960b9f6a1635da4b647c7e61fbe7adc7a91c1366e5a7cfb492d833cf7d6127d123154445ff19979908c3dbd4b0cabdf6e5406690c9479f4cde0cb0cd2d980442e2be28e176c29d841e261a5b3e45d9eba698b5a7f44b94bb5907b7007e00641f6ddfacabc5028c9a660c015ffd19e81b52c6732fb9cd89e1e0c2c577d890c3e048eb7f9524658298df43aa9e152f11fda1e2eda74ac34f0051ff12b6596e7ba4b5b1cf9057267fff889a918c71bcc50c1a6e09e6aa55a7478bf72c6c6954e0a746b0621fa1fae6f09535fe0a4991f3a2cec769a823130c0e950d4740c2577e6df7bcf330d305edf8e80af521afb5dd5c94cddb715f54fe38fb6ec0a570a9475ae9a6a3aa8aae36fb15711eb09ade6887f8d347532a5b158a8fedac6424c3133d1a9396d6e355964b398b8aea63e68bbed134f335b6280e9f6070be60099af4992b10eca6beaf400e876ec23729b6686b7da637b1fc80ff688f850a8b51d6a59881f7fb114318ab85f4537a5d63c1ac26480cfe1d6b41f10fd401f4a5def8dc4cb4312b740006998f0ed7766580eaec26f7b18cee4f69f7867aea8a9eacfa63200ffb9c7dfa3ddbd5fc612f2844d5592b004897e8c3d7c51670464b75c6feeddb8f8f18f9c1e2c5408807aff0351316f8a143b51b688bfc8f72fd693b192fc1aff4614fb2da557d217e7cda5dbe4725368c886c69a795c6d823ba2b9535c41f6ae4716a30d7948dbc8f65627946d577e3a282786485f5fa73ebfdc63f53321759e4681d53845dac77ebfced9c530dd15c520797fc9d5fc0fe1fb53eb04af1e86eaa0db2b080774e5c902628b3a3d6df445cfa17d06ec536e79fdebe144b61f3bbd269f690f0617f155f3f6792ded800f9ede44d0b77951b28676d9d9898ef250d67e9c0bd71880704b9b1d695b3dafa9d165de70a285e38ec927a93fe2ef4f0ac57c2a622073dca3c25740dd55f7437036505d207bdbbe156382d841741a94942447441be17d95c621bc51260f5c930f4727308cb7e1bf68611a275b5f8da545279323b8096678ba84a0ab805b3162996fb34de22e7fca19820a893effab1095862dedfd711ec413769cf84e93bf71397636efaaf16915f89d48f2ad4a5d7c473901cc350ab47554a303a04efc7ba02a7f593aa45da4bd66b70d45c0863ee9bb5048c894bedadc5528035efca372af3ccbbbcc20c1240a0b025c2e4ee0f5eba8 +erase 0,1 +erase 0,4 +erase 2,3 +erase 2,4 +case 16 4 1300 lcg +parity 0 85f104a041c87405e95cc483c09d776675d5820ec5b2f06cb467f6603a6f6c28f82bb7e12333145d30cf956f6de66d56b8a76754f2b8d60748dae4bee952cfd9a52538e6c13e6fa87b831084edecf3b4e396c1cc7b8eb2abea93e54e63a65cd71d243e7f846cb23dc66df0ba6158bb1b8342a3fb4903083c0f7b29097c88a19a57a98aa959cce846d42ce6214ad3015e7dda73c6e15c1a9b302954ce997b31486be47de00707f9e164ef564d38a12c156205a8b838b11e403271be18a1c6749769e7752e492146dd0373d2d32c4fcc591d128836b6f7e88ac9ad7c5518f9ab56f76eee76fb4ac8e39d7cc7b9711f06e8543e36ecbc8b6b3a8febf0c0424a7a24644a958c989abc25630011444afecbce36b61cfc7098d56807aa31cdecf3ae7d481ea9f8780bb5f4205a936d4bd61409e13cea8db8e7e2c44fc8d9197a9c3131e4edfe9127f1057cce7e42003908410c4f8afbfea530eb4a995299a99b938457933c273b131ef8b7d09b48767e65f4602032e821356a8d779a0f890f7d773326fa3e2f9f79ef71db0690d0107fd6f73c546fc7a3e12f862f2285eb23c598cd5e0bf1086c1df709948f031fbb351ac9e9e77846eaa598ef6272366f0aa667e781accfeb6261225753d5cc8d20415922c8a8dcc5e89120db712a092b6d3722dd9024bb0bda72c83d08527e3db8c1211ebc729279f7026d9119b95f4acb3ca0d2cb4984b0a92265c756feb48c435ad438332b597620b7749b21f3c5aa2ae228fa07fb46f2c8ba9319078dffa9ee1dd4ba0f181d94e367c1596466355f76ac303acc3b58bed365361a8ddb706746fdf595bdfd4e53d0e6d24a37b416b92b09ec173c091f4587889527fb505003c5ad3cd6dba9f56fc163c2bbe859a4e99fa4484341461822fe998b2a8866ed93918a686dca1608b3b3126ed3dff8034d2aa9296949bee7048b76204a65e2d214a096f465b38cd69d59b5591815d6f70c54d904abace5bc1d0e6dc4adb0c5a75b0d3f417519b3921aa63216eb69caf8a41ad29c1f071d00b6ce968155b3053e4dc223670e8139bdad5edb9a2c6d2dd1e4bb8f12b1bb534bc5efb415df36b30a4f3c71152abfcdccd28fefab11bb857bfb777cefe8d7f6a91087f6a0d31b52f33b4e66a8177fa0349d5fdbcdc84cd1b0f0b9ff6b841180bc746ce4738661ffc08eaddf1b9ed2e1d1216a0b9c741c57521d972a429505ce01bd54ea6191a5ebd495202e3bb24609b3785f21bc64cf9ae3bb0df13ded758e501725a73d6df6c89e1eb3c15757a88646176f0ab32392327ac53416b67a17d17d64c9471ddd3f083e877e50f1e335bb1a887e3297d1863414520a3722bbe3b9952b0d1394454e71622705fb49411c457d5003c24d2325b8407cdabab97fcce97c056fa6f516cbbdb984397fb5d80b0d6680f3fb4b755f53a6118e721bb0731a5f793651b16da7aa9a6f97fd7e8dca561052b6f85b14a6736bec6d9549e35cc90dfd1a07c85981a10cd251ac69b96574836a21e3dd536031fd31108d5f572a58e333bed3b9e51a149e074680bd0364fb0e7f71f40ba3f33f4b156bf54ad4f5c0d74c2b41f94f03f11ef93d5e4ed92e8d24f9b13295178f65c0761ea507b35b65d626fc452378c3fea7e7d2c24d8955f1b5fc7c673bb25772c9910b1c36cc0ff770f32002c54bc52e3bded5580fed0a16eaed351992178ced525364c2c38ea405dbb318ddbaee25a157e11f5525c3b073d7bf0069f1bbb8e4df3005c5248b70edd166716eba20390bb295da0135916e406e3363dddad52a4e2ebf47e242fb838c213419461afc7dcffcd1cb2cf546b9a7977aaa +parity 1 25c276c043503575a53501e07972c6fa4f61faad0602b1c106c825f32094d08a469bf5cc5852bc50bc7e30ec60898c1b226edbdacd137973fa898908bcfab88e1ff015216383523f42f7c986b4f89b6ea6f8de9e401a7a865c36924723a85422c6986eeb551aeeae0696c01028f7226bd5e79d42ca2fdb07f1e668d45f345bcc8a30b07a24bf8bc04c507b90462f3e2cec768397cb12afce61252e240c1718e39158b0b2ecbf9391d43ce60b365928bbed1247957c53cbc68fcab81a2895c9021983b43e36607bada430e87eab160a23d326a403f99278894b5cd05d8d70b278150355e42cb6c9e2bd8299c92e9f1a6dc42e2ae016ce606985116fd98a0d95259ce983603edd896cb307e3549c7741b554f03eb90725c82af3ee82d57370f8922377a5a06de63dd828a6546844fdf1bf830eebf62d13f3904360b6993a7ab11bddc7cc497f7e426555ac679b76dde2acbd05c00e3045da6422739778dbb94eb9c79f3c4f1afed83a9f73dac2dfd69137c756368bedc18b51244161af375920608d1ab84189f44a71befab5d16982a3ba2d3377f2de9616ebde5eeb185b835d235a2d014629be593fa61022bc875321474f069eac9441897c8d262bb767b4e45a05c02a889f1b4209e69e53726d429ee59f404dd995698186217aa8d61f53981c599a665bca8785a025af491c47e869172e65f77ceaea3c7ed6399115ebfc71ec2f74c2122570dcc79a77a82b708ca810d1511ac2beb887752ccbb10b7d8e02b6d8f347a165c09cf604843a5ceac51a466c0c07c9ac2100cb658b82acb48dac98ff0cd4729943fcbe1dd1447e1fda5879ccf46c6acb35392a665853219b9f888d4c6592943cc8e5e7a1881a8bd8f319c1c58d54dd4850572a794f66dbfd86ceecc355c6a2e0557374a921691183cb4da4ae13261305d750f44dd5797867d6255ed31f1641f097af5e9e3c94e779629ea3f23ecbc3ba973323670e7fd506f07ded135a60a3d27f5d86cfcc91207c412f53df27745bb603c4e029b3dc6438de42d26f421ec718e34b7ae90a77784f34bb1e15e1def1fbfc661d3942b11fd215a568fc3fa2771b66c93e0fd60da311bf8a4189a2a554c6c68ae37ceab911255c0bb2ba647606e3df65f60a419bca9aa60e600d3f929501cadc29ee17b2ade204a07ce3227b27e2f993eddb8effa59c8d3a235d9eab90135a74adee00d99b8155b1306e7246962948bf2c9804928356456d0dc5ada1eea6ebd906f2a4c9e270dbea9cf6ede8e359aa3f86b90cdd6f2733860171adbc0cc3914b0bea6bd575c3224002ec90842d7c570ef3ce0db05f980e66be16295efeb4f605372a2bcdd20e955bddf8e02dfe8c50691b6fa923d1585c987a0473b47b1054da3e8594d9b8578be77fc9541b0b64de511818b1558dbf079af95d77e3fae697db8b9f186a3c7c8776b42b1c4205551a2002c3e624d4cbf9fc10368a76d2ecb25f4c0126358baa26f261d9d6aace6125ba024bec6d916bcafb92480ada6a15e8812969299fb70ba4df4240a7f2bbbdb7c5548794d25d11d802e0687185d1c0396b422cb7737c35544289aa20a9adb9c1176091109ca1ce8fdc198275e7507b3b2adfa3b23dd394e1c30809a18100e45187c2518e89fbfedf98cef84ae181a7756ac90eeef3e3d7cb6c5937af3d3c21f51b04a3a89610905f3e33c4f2372dacd24a41dad16914f9c5de5c6d3bcb1c6b7ba351d4f5c1874f62c81cd078e13f6460f254d77514ad36340a7d0ccb7b674e578b049d95aa9994da17cee8a60d1c56997ba6df9850c61ec88987c7143f86b0bacd690840c8e5eed3b6a1a982d4be +parity 2 c1def77fb8d48ef70aa21ffdc7a53c19535c94f1d9d2a4f9418ba26038abbfb4ea681f4b9b80f897cc62547e45bd90b3bb4c2d349dca16d288ac5dcaf8e54f50cf133bf425ec9e81159a79ba542d08e789771e444b682f52cf1ebc92883439f410e67e374f925d8e9cc80e752a02cffc546df776090028be00cd43201da2a7ab75ab40d750dc5dc2421ef270b66b8ed817908e4648a6d71a9a108915f53246e775d7fbe3330cd50a10686dcee7bda7d59f697bea302e3c537161e9b45ee096c42ebec3edb2c15b1de7429f090cd7f305b3883c6afc5a434c0abe24174147c5165dd3488e4f506a25775d6af9ffe608c338c3cdba0180b5661018a4e61e21be3644493201881e274a60790c62c907d884c03fbff6a11bcca79aed297c103c28793ee1dbeae7bd27eac73f847822cc6b8b3f0e382646c7086feb1a794c1dff6afb3f42d536a10300a27d659f5268991129da95818c7dfe1bb2fdae912c77417654f61c7804364f8471467f2954a1f2c70cc809a4ddcba86707e6dada3bfdb51eb897a5f5800434f511e47cca68594c2d33b248f9c80eeca626256bf52a79d215f973e0d53ee056326fd6a514ffff012279ed1453bcc47899d9a6de7f2c3f91800be81c18961100db07057eb5796a847ccceecbf6ee09a29a358fad7555232c0efd1b264f494be713767e05c507ddd7344633bf6ef9492292660632157275ec03317a015b7fe096118663bcec9c51710f2d6b7eb52984e15b70c21de6561c69900764f11bb35d50dbccf249f9dd3cba050ae3b04866e5e57135048d7620de0fe02f2e78177ed7016c39fa6ca13a106725fe002ae916357aa05a1a33dc832d728faec96c1a2772aee8255c6cb843f6a74c544eff20fcd76aa9b2812691dea9253c6d9ece8f9f9b30d798078fc77620dfa47a3319cc0c21012addfcea1b6bfbc3d21918118441812b705082dc1784a45ec935e7f780967315df01edfcbd282faffac95ce541b61015c91e2c7ab0d043ccfbd8a0b8df27411de63ecf5e1e5c464c7d22a2d55f86335ce2becac2a2da8f18c1f9ab4f9ffca28a15f8435a749a0cf4e305fc1a63ac4d6df5ea5e70eb7e13b96a427320f8e688fa8134ac82fb38acdaa9166dc7828301dae3ade26e0125ca1c57d813a41f837574f78dc4bef94a2cc249f9d6f7e8f69350219a64163b1c9dffbdd01968d43e8d4b5e398ed0134d44ace13de4e62482de0337501406e468d4db20e3d1c92459002034984f522246f90d8349be1fc56489fd365f18112b51e85718e104402b553c4f341fd72ef70b05e42b0c0e50fb847c6bebc68978c9a26488494b080cbd4268ef7c0c834cd46654389af554c1fcfcaaa9aa11c87856c57c7847f61e836738a67ef186c7fadbebcae06a8b3ed72f2ccd09bdcd92afb44302f7f297ed0c4a5bab9a7fe577d95bbdf41108ca06b4fa73219c0e29fe8b9671afd99b9e380f3601bcd722baf71cfc1e57645fa38ca155ca1de08a6f655901d1617ed7e286c1f13e676b4a6ffbaa3c21de9a6249f32cb3708390c88da2e460648b67a297d48df8963e9ce1aae79ab6088cbbb4ed242687b88143303a04b360e17365d1a7d99230b5dd93ae498c622412c8317db80c90d0d8e619e26255bcbdb7e318457d22825eabb9216a242d87cbc484722a2e34c9bfb6f6bde8533161a28e5afa0a9d61adb70110b21fdd53076cbe33a9787081a05210fbce1d2f69f8d89629577feec55a4ba7fde1013a67065a4e4eca628d37ad8e54b6ebde7e4b32467fdb5c77460cf2ec36fcc861d81e425d53eddec67b0c47dc61dc7d483bc51cf27d4cfa9b76566aac73 +parity 3 cecdb160ca6d4f767f38aa79601a9b78123c1f6c57a032264146ff7ec479dd12fa9c05cf69840fee3a05763be1004d561567481197dbba7be318b51724c17ddc8fc05e36302d2ba0c38baa9df984b5306f35e65701e3deeee7f6e8a8185d31bc7c6a56880c056bfa8c2c180e8ac67216cbe1dd60ec7eec794bb9502158e95068eb34c9517d18f00925e21bceb90be0f7e2a6274bb0ea8ab8ccd8cc8cfb5402ca5e91253acf0fd00272e3adebc452d5b67278c408921235dbe9b867880e13172ab31f4e660217d0ad83c372b21f745353da2e4e2d3c1818fee7416e8877c262ad1a9171fb5b424a2d57ed87bed4c7c00b5f2f0d21beb2df8cc6db6315d0e69daaf9acc2f32f9e97aeb9ca04727abc4035dcde6715a3827a8c85f2398115937eea3a707ae5ecfc81c50ff635a4b7440ce63cbc60c946ca5e1c2404fd2dbb5a3cfed81a46dfb40bd2313c49357d6f16599605796a4246320e17b2d7be8e3d748dc088fb42d3babe46e9b084b9f6857ee950d4595668ef46ec5c0a14f1132e6e5e634c74ab6881957a94b0049c11bf34b9fae8b29d54c65336ad7c5fb83e6ba487e2d605f69baad77ef99aed9a3c92d4396a08979ffc50dc04ddcaf8b0ca0cda334f81471a000f8077e17de673453d7c7a6400edf7c2b6b1bb7a3e21e9e542ef97a350074b5cc1b24c800fa74d53ad5b169c3a0a1bcf1e4a8d7c4e2d57db6099c85834735049dc61b4e6532baff8b0889f9dd60bcda8e71d1115c6bf43b3388b1f7fb698ef2b303c53ecca034c40d27761c671b1e1f18501f193842d91896603bac7d39ed1df95e9cdc20133cb356293dbb202942a3d31215a8136f7745acbad849fbb00f540e44e8eff52a31802723437329a9c8f0915290dbabcbf7234aa2c47afdc4e913186b6d02d3861bf1c03343fe4a89b05657815ebb659430cb074ba68581c1cfbf1c7917bd2ae4bd032f0a6d0b238f1aa77dc03eaf6f84ea28414a2ee3109d172a8d689bb6f3b11997962d839502fe7b615df3074467132ece03a2a889b10fad08420f0824b7f17d4d9bb3db05dea51d8048c52c0415dc6d21a776901121c691280db8892bd645a40d66a55ca686c4e1b258f7805b4f44c93cf929a2061581502ee519ee5645c267ea129f7f047279df3261c63a5c055882b5f1f82d83030916bfba8e543b59fd6d13b9b0e78b8dc2d1d9dcd20f6423cd17128dabd686ea93681a819cdf9ed89c6bca0f82dde6fc79b6978a4a3e52194e23dfeb3ab581fb7a01d7cd72922b274a5fe77f03a21c1c833f73661b0e3a6abeac47d27932037ce8eb4810e542645a534c3e020027c902c1592acbd7ff806cee106077491552315effa60edb2f90069ecf1468d0d1a49f04c91fb3fed5b6c63ebe3cbd181c99a1e9e33efc70b15f71e9075793032bd793b1bff50e759b319c61c29d24209c91726b11a05cb6410e05bb3974cdfc2647e622ddf062e8559e870f5bf09a32a77d46245fd8f02cbb32ea0ae1a111d1060bc764b19b8d558bb319f9cd4f278a17d24e5c1848bcdb2a2f0c65ac3414013e72c60aaed59693328c6cf8f0aead9d87028ccaf1ebaecd0905630e6b7b846790936b9b6983ff04c48a65c37c88b16da6ecedb464a1aea2fc7318791f1cb6f0bf06585e2b27c21524cb4a54bd7ee95c455a2675898d4d88c81d6ac465a6d2f98186fb780b2e1e0836ffeed84d8348f1de726d9bdd71415a8d9fa1336851a79ba842188b4831911d4bfa02a6527b6c5a5413a6858d046d25bbd14cd5cdf871e54bb8f9f467e433f4ef7585b6dda0795be4c6eeb0eca7e2ff61fd259a2e10485f5bcbd31d6adf1 +erase 0,1,2,3 +erase 0,8,16,17 +erase 8,16,17,18 +erase 12,13,14,15 +case 3 2 4000 lcg +parity 0 9f31dd8658475a0b0efce438c7f7591a0596afdc19fe90fce15355887c7de3ca42c39466a9d9eb4cfbdd1e52c3344b99bb47f88f373fd73d7e1425d6d5473112158810cacd6c8db651345d800bdcb9742d15aa909051ec9ef03039d76ffa448e35c78d81584d1cbf16598f09c1bb55c7fa5d73f73d3b22c6eecee2e31b5c7873730da9392b081d3565e9c35f4eef729d38770ca8e9b43bc7d24ee78ad9062c1bebe9fd7921d54c8aca47beff725d0c814843a0f0d606ceb16858c2a2578638d50b27929c2ab4f294ed0a5d5d762b6cf2143cec2067684c415246fb2adf7d32a2f0668f0f222a090cbba0a2ff9c0f0e1a364dbc37f5ab633b0b81a80f9bfc483c0e80d61cdce6310eaafaeaae1fecd59c6de2a730947376f11a349169c7b8e425ef4edfd46f2fdc123ffb7fdf5eed960af439b7bc6145c642c68eaca69b243fb3f152ec5b02234d61ecb8ac91a95bea564a6c30039a26868a3c7437a8299c0eab2d4b0bc9a32167f50f330be99228e4c2f67260897bcf719eac2f323a435f85138870fa16e517e5edbe51b94ecfcaf754fa62a452accaa723fb7fb6f2058ddf0864b76b5cdd0363b3ec95a551b67a63f6f17d4b67e3c796789114dcdcdc25cbcaac28d34dbc9d05acd17e2ddc594d32c63227c876c6360605754db1b76b5f7dd3ff86282207efc18b2eec7e345c49ff4068f24b1a83a7122f48780175d492cd0bc16c8634d286b39765a19ced55e80f2e1e0d7c6023372118bdf120642a97922826b1a2475fdb3b0c2f99df82721db40d7194af59da1459ae5e23f5c629a215a50ca9b225d53060a90b7392dadcff98d479c996ecc27c43b023c04f9fadd7abe573f7dbc6d6a8a8025952d96ae4a42dea867e84a05c393668ddddf46216807b8afa36babcafe4b7140123b9cca9f4e3250e0891124959b948b5550aae343edc7e92b0cf2e40a8def6d86ccac15f0b383230cd74cc207888fe2d502834c96153c2455590a543c7ab43ea278e7a67064c7089e7d8b2b99e1e96cd0a6f68f0f2b0312a00c6fc16a53b3c7951f2f293fc042dffa3ea2d76ae074d2fdea96674c4f588de94079c1a005622ac6c268100c70a47f4c2bc954be1401b15652fb8d6722a91de3ffe0e0aa34d6baa8e9d2b616b2a9cc53ca0f7548dbe3bc11dcc226e13d81275ef0bb84336fedeb2f430a1530424de1c1aca9bf36cb296645571cc5efb77637610f0a12cd1d1d999461925cfa50060096972cab9baad94e140f570cf97867cf7a3e27ed80e6ff453b498fe30aa6098fe1045721b2d24943ce9e3c6255dc59c175d7bc14a3cdfd6e8a264ff97dfc18b955cd806486cd41633f5f19b917d4bfe7b38a8e2913f428ac4868ddfce7304fe23fe741a89c3d2dcf21b707b857bb4d614b4a22be79004df8d03b4a837b1324877fb541bd2d262a703cb2622e82b7275c266900b73362e753f80f0e9d00b31237de97763bcfd5e64721e2d4cfc2387402a0935f795d87ca7f22875aaa70b72765dea646b024f64c82c0187c87e61b4197c2a1d6f135e3e48fbc6fd5189033ea9c0c2f31fa69ad9b403709a97e0b95934c5f2f816d2e91181f7246bf37429df93bfca66232232ac9fc9d8aa1e503e942baa2bde26aed45b67e0910c9720ee7a13590733b92614a418060798b06456fce2334da66ba01f412803a432b51d6e7db63004999ec183d1909063a851bd99ad1c57dd70958c3a01a873f6dfb8706f1ec01940e34dd11664b06fbe27542a1b4ba7543d1d4614e6a113d90ba2d4fd1ed531eb5d3728f2dd9150f448e6466b49e46370204065ae3215cabcd88ece25c9b106f78dbcf3787bcab1444f656e097733a4a90206b41ba44e5760d7b6708d0137702ec5d491b29f9c12b7a92d2aedc09d326a45c3729b977fef91fa723ecad3b68574ba1c8662394ce175a37d853dc47aa5301f1d9698b3f923a4c0f39a49d748f4ffddd1dfb343d1924462d6dfe0393d3e44a3447ab8983126a043006e11d89e123f459441ff63ed40b880d9bc05547a5c2a6f38509ff8576c3be4952133fabb193e0c9f05bfb8b3e7b4e6c1265cb9fcabc6e029d44be54d18c2095920420f1cedf1b72999c1c6abe20833df08334edbf4f3692e83d288c6f30df4bd63bcf138b61bd4e1a33e9b6d688e7ff396ec965d22089c1c5399797648a491d312a6245bf31c1183459a7d1d0cf6615a26d31a2ea8abb0bf5d1d8edb0695ad6be4ff99cd909ae1b4d09b69b40893f9e757fc02fe313ec3cbb3c7229d830e97f3b4d274a560f5f8d2631f269b6843fc11ed870618963b3b62f2fd6ffab5debee09cccb9caf32968db79d47b2fac02dcf899f80422b45c13bc74e34dfe10d2e8f13848d792d997a82f22396bd247423c2f65b07cc52889c97bf0b86ac68a504f0ba7797d30fcbe9c69adf9055079727461e7fa1d1a771387b79e9842b094a64d6d1fee370f423997472cb014563a21602b915aafbd0db7998bc33394e0597fa8b30f8b63399a2d8c2d8d0a725b59a53437317eff5c7ba55db4a32826536a76b1642ddd2c4578d98c625530c14043ac797edbe42d4441197e26699b2f628bc273484ea01b2bd1bc42d3c6ea6e81c5425634d8035c6002e7f4abea71fd1c2c3833137ed2c515db259c2802e20385ee2c02f6d84da3a298c9a970ec8217c6500b166c36597e7b9c7a95caa722d1481cff27ee16a322de219a5343ffd936a58558067a55a35ab0af43b65e44d8ece1c664f3e45a8a262fee1bd37d1c9b2ec44c8f7935bec4f425f2f50207765ef3ea3e67c4bcb0a4c2d2a3ee23120ac5df410e004507433ab6f082589b5a006be8d4cc6c8e2522c5cf62c37b543540263eb133ffd81d4d4be53b32a6fea13024e6bb5809461f9d052ef84428b90f9cd746b3ae3d6cce9deeb1ff736dc3cb024d1389a74224b4aff36a863dc8f54e3e24a725045ebca846d3ed7fe052bc147aeef80ff5aa8272b92a8348d259bf55f2f24df3373448a1f54b9d5bd5c0e63495c972cdccafad00c830dfd9e27e263ae24b4e59f3c9d88270d834d009295107afb97df2f185f0e12c3be20270e7c83c0dbdf287fd236cd91e46654c91e32875864f45503cfe1f387ad17c0194ca5954f43b43dd6105e82dc8799ae68c9d19f73e46d71e51b49e33c05ef8b18be7f350b79a2d7a9631cfeb9c890f4d610628ba937a166e19f83f53700ea256f59c9aca8110bb596a86ecf15a59d69551d51903ecec8fd40a7ebed202efe63f0dca5bd94e474433d9ffe08c71c11b60d52b43f7fb7d46464483584f0d74aa8b808fab854e042bcbca5eb4f6a378ee94b3b60710017d19ad8708309bc2271aa3ad3dfb0b17100e8c8227c0ff615f5d46de82422c4c5395d5ea59e752ef658c995c4a2d6fa14b7ad4b0adecf3d3a1f3da401fcc62c4132eb687f98c2fe65f1dcd55e63c9f08ed64cd13ae367f572492f8aa562daf6ca92c1d9b934538c6f85b86be1f981f71fdae14b80b6e5248512a0a55e02696108541746b206f253284d4e994bfc04929f04568707fb99315bbe5b90e0dfa3ecbfb431b3d190fae608ce6df3e6c76d35e4bdadf90653f03d2c148eb05c6edba0ac28e3f12dc73eb5e023a3a634b0b13cedbe7ee3152c6390db621a832f0b997775c80879298c1ed24e026984e27411385817bce391429659c78b610a1ed29cbe2c4d02f30df0d0898a7ad9166cb5d2a18d48b8590eef75cd564af25aa0d0e3f99144feb2680b2e3fc11162ca1d401e875e26a41f235898df5254b19f52001a979ccbe7fe2d10562469c5120417bcc5488261aa00efd6ad751d2672ee1f073e4ed10cb18b0d2c05f3b4c9c8365dc3fcee37e00f66aa6f1ab6b32f0e9dc51e92e309df594b7d2cdb108c33a09dbd9f91d8556be7e17b7149627687e027d6a98a26d01e1e45a98eade2916fcb970db6a671f0badaac63160594a1446ff6e0fcdb2a95a3a91e3a3ad86c257f114598f49258940f4fada6c9de3c92b25ddb473f84d18357a9e00d94d43ef84a0cc6d7a60ed816468b6ae8fad19cd4f1ee81bde1cdbc8e3c7df83e52bbc94d650c70a1e1d79330cc22bc5ca3f181db525c9915d956f9ededaf3484c5650d99d90b98dcf41767e515c276a9109df549f0e640c46cd5ad59a034f382e6460a5185dc6aba39b9318e19579496a3af1f88d1efc93fd53d1b0914415da71afc06feb727b51e58e4515d7cf5b40fbe1f44caafcd4bdd2065ff99dee099a7bb07842a9a91dee897794837971b78d48f6009221c9fb986a2b609f18ac11665dcb30fa36cebc3bf438007ddcb4ee59e8617773904da76e5be9ed441b99412893f308c03379230bb5ecff01c28ed31f5048546fd026d41c05e3006857e410c4dd66da52eb1628b32b8db84605ce61fb4506ee42988aba1ffe3a0b21b6a0349384da4b54571e9bf01093784976a271e29bcea4959d4cf434b528489858a338277aa73abbfc1f17fa79e024a3b4c0ad13becb63d26e520a08e8e31d795947a67262860a2a14cee09e2f9e04ea251b864d403360aa360edb68caedb5f19437f2dd60d173ebd40fc64767e00ca8e3a92b783d0ccb3f8598b26bfe94878da1207e3bac65046063df1a43fbfaf33648a0f6f4939456e80b766a2f4e277ee243cf63fbc2c147e743c96d4d6abeecc7c8105eae84c7b29f94430abd9749ff752aec3d6504288fe7262645543c5ebad8bb17194689a64530a5952cfda40aa7288c04848d4e91867b8035b36896170d40af461353e6fa634351c784cc05b05d17455d1a41495a700a51b4f7057477eb7fd6590c91def292abfab326733925f9bff8570ddac6d7309ccd99c978cd7ef7d6b6462ba9b462c3c61ab69446d3c1d7722ed7c6270de18ee24c3685d6a5f7e40fb378f02a97ea2dd13b3e6530adc1f29a2c98cc0b4409847cfc8382df7d062a19860b9f966584fa45011a27b49868b4aa3e7c0cc17ab838976efbea25edec1d4bba9245580b7a53711e042c111d3e8ffd3b5cc6246fa6d679b68c0aba113ea1b6a5080f9ec7fb1a1e48efe9ec52b9c5df0a5a752ece9a33fb9047c89e2166629c7268570af5a61fbec1e1d0eaecda14920954bc7616ae1b84d3bd2b5aadaf967fbbdd767c61549f61892a434eaa9cdb9fe5c1c1590c023942fd53cc05cd7ab1e75b577897dbbe5a1b7b2c983d9f9cf3ee699f65011bcd1c02ac4dea4ef91f3b326c6de2c889658ddcde04eb11f8aef94464060ae13a1fbc5010556bc588cfd7f51a1f19f6db6cff455de980bfb18960d3524bf85f7d6d30f4e8d2d329b40eb8367feb5b9d3f7ba70ef84fb9d928822a49aebda04b45f0705702602306ce437eb20da3be0ccd00c2372a9cc6e7e82d0faf29646efec02782481c7f5b3dafe069727a6e59a9809622dc02bca681061e2430b4fdae71b039f7595e38045a8adf6679dd2b632e4a2121298429ce4b6377b79254f6fd3cd25895c1627f716971cf38d6b79af342e8e6c1f0732d59926491ade2e8125db0341e459dea8290dcb91c4ca8279d3fc16b80fcc40495df02e3bb9a48958a4e6254229530bdc40c0cce50bdeb1a7c6ff6f85e289b24028cec87f7f6665720fa88fd628598ee7a6a5b6499115c659cc01e8caa3e2abc3457e2a23fd23e57dcc52957608a1683469fd7 +parity 1 8db62234c2ba4073f6367b9738b0fb00ad4b5908c5c5482725cbbc1b83452c75f9dc380860d5bfb4d609c18a9bc253eadec25917fb990133679b06eef185e7869772f129bf49ce66d26e949db10f7558938e89be24116c9610b6cada1bf07ccf6fc516f1da131d2f3783d5ae5a7c1289bad68b8c41426548e065b303d438db30a26669f25ae43447ed055bc8d760cda1bef3b883e25e487e245e6a8ec994c143fe0dd40163ebd1bd0b9e72818a7417a875b466ebd063b68edd717a65f1b88f0f0801658ef20d619558ff195fb93951c86339d71f4cfada3e7f20dc9bcfaa5e621030f05998d2222bfca592e0b77a15effdd41e546f2fc2709c9e994843006c70b2b2ec29ad51566c82d12103f9f3c314f1cd46ed41ed297cdde81924326b08d318ae4116c6a0b36756a1086cb99ff98b89f89deae44c1c55b6b00f270f60cf041f73b8b6dc978bd5d156b506c3d6f71688e5b5aa74ed5a492be0c20bfe42c8ed3f56bdbd980ce36d85f2b73a90c53247cfed3496dd304f5f58ef3043e5ba375c1c5eade78e456db3e88b2ce64bbb76d30897b8e5f86cb93c02f44e9b0aeebb619efdc0b14de115579b67676159a504792a4447df623724c2133aa654e3f7c1bb7148c473617fc3a80ef1f34e087bd352cac1fba27b232b273b77b3fa950da2a46763f7448f8e706269edff5a231783c5d355a71b9b9cf00490f0515c6706ddf75b82b48e636f75f00a73358bac2bb17f2970d3008212e033b030f5e77ab9845c0c4e7c4e862ca72c19f43e4ce29e02a0b783e61522793cd5e6dfbe8fc79456594ae58303e5df390fb29226d43ba767e1e859ca6a450648c406182f199c74af7332d48bd7811d833460718ea16c39565decea6499f2bcdb7ca4ce6183f167677aac718610d87e8990085e2cce7cb6038da5641be5252ccfb6e09062cb9c49f00fbbbb14ce4a315ca0bb81ff122601d91aae177bed28adaa1bad50e0749b6bb22e21345fd16456ca84e28546f0c2b422f38523e986061613dde2730f13b514635d57e0754f56a6a41db5fc680671769e675345394a0a4d5a96dcd578c20d30d2c51176a7ad2a41e429252d54ef5a8964445a70254a68fe7f87e7c984a82319a0b394697827b4edf578114b786b228fd5f235f3a7979628470caf97690398211a383099224595a1fd6dea45a07374a1cbbda2ab81ee6b09079bebf48def800c8b7ac14612fcfdd4df765127dff0ee2d51d586009e087b388eb7f875a368737e07efa822b708ee083d5813736dc815c300e3a11cc39ed3fd1f6a9f8783a430f5410c501cf852f31ce4c3c3925d8a37198fe79a5e254d185c3ee392d4b806f06c0eb8a1147901a60c43d39382d7c762ff3bfa1c68a428775e3e12d22fa11b4a5cd06e8900ca488904afca4bb07574c8cf2230e5dca426010ca1e5b6d0cd2c206289ce06944625daa27213eef6bb5ef1c4b7441af6719191c1d511e07d3ab5edbd1ef7f6e135cb4291220849ed3b77b3359e1491a7df52883e86713bbe5b14c9f6dde932fd2612ed64fc6eab751bf1243b79749de04a2f6f8c1daae9697cef2144120acb40413ad598854ee06df159c631561ef06c011abc7d651eb8c94db7fb847c88a6582368da91a28599363c638b09201790d4c9ef3a9d8b3fba72490dbebca2f844f26783890960b32b2da80358a11a806cad6813d0ed20edc4d95aa39e31ce6775112f21cbf71bed04f43f61c9a188a8c080c25813b9edb215ba41e35bf223aac9253f3384e7f6c88042a72d43871464089b8a9b6d128a00f582baf30a14cf55793b51f10a2958f9732c4da748d767b8a64eb4a54d1d77d9ea7aeb423fbcac2ddb23f6f68152b6eb371749611fc2e3247511f8e9c8fed5154f43cb8771d6c6ac47b3a6151662a4c328be30d34861d5063e19b3bf2ced7a4d68983fde57f27c2014d7f0f056e880e7db978ff0b2e57909201f0718c62607d141b8cc3f4c68a5defa27dae07507a2832be5e044d24b87ba6921a2df8e8f6c99807ee99c9252ea866f402dfb963cead5bce5dda940c13fa4f2cac7e4cd48b8c967226618bad6cadace7853191e2d09bf68c1c5d19e93139822bbd94e9df52215ed1d06396104173cd5dee2e7f19f8c30a9e599224601d9b2a27b0333b25eaeb35b212f9afc9a0312f581a9aa9b30a71054d8f0a24dc2878b9e0023e15b0dbf884ff2312d64c5d97077659798defea330180e64e23b620c03cc1d85b2b1504e68aca31bd83139d4d3961efa0efc22c8550d73d43d44d917ce9e860309960e866c31a6faee795b002f8c6a0d36c2789555ced1e81f40f6dd1954ac782a4eada84e65c9aaba5f9eeadad0c35684cdbde8f41cad56eb99de2148200ac3e3c2839baca892e8072c93d1d55f43b2ce57d396b964a1ddb1ef305c2f7f0c915eda2605c57ad2d34e99e2ea97c8fc5660d9905a0d35d1c4e17be2645fd355acfac027787ae517718eb5e256f937047bd43acf0215273595aa3da9ceb3e85a76296837c53172c0d609b828868a465cc2dddc7b54011d3467c1f45d534c7f71735dfcec09ac98a2989eb2e8287c37b8a4f53e0fddd642776fda6b601d2ee3e4f88a4ad04cd4a82785d1f21924346138ac773c6688aebdd2aabcb4a3ada64141595ad0c7fc918aa667f1f74b7102b956d5834e689e43ac44f826f1fe6d3734ce61954669b5ca4440c9144b3a0374141fdf1faa85bdd69a15f99228a2eb72609a1b3528e2a432c1fc29d008ad535c6e9c1339414d0105f00095e8b5551326dddca3e5892fc32c8eb22572858f0784194184ccb666f9093a6278ec08cfb20074701f212a25e0717ac19f22deae00126b886daf24a4389b064857f0d6269199d05d366a5b6138bcb0c5998638a75dd17b793171ba196f76de4b1b234375a0a4ca211759a8cecf586364740a963572e76589fadeb31af2c233828a33f5a05887269cbf690b6fd69b875af257c314f7ccb173e64c6032fc7e92fc364017d55f30ce22f379aa8a776474721c1b33ddf3e38e0817528429a3aafa0bc97e221e9f2ec36646ff3b3806bfdc59e3f212fd1fb36f0ac1aadbb10894acc178ddfd8bb64614ba6d9d14d39c27a4c4e6bf14627b0b3559a20e05439119c000eb0add9b2cca503104d477875a6ef1047dacbf47c163234d55c6f064db91c2de73cdd16ac35971d0c5d24575882377d09b0edf776c8acf5d621ba51c4b24e6c25269d4e016a489ebf2ebc8d0b80cc4e39365f46bdd6b2c99a532025a74e8e4a72b1f22dd38f57970893183405114bad8833a93a8a55309f0a24e0edf6a79b1693e9eb4394a289966433c600a9cb0b3ad9ac730a4d8917dbbf595daf415e5da6f869420aab0d1f0ef25bfe0ac102cd04eef5ab4b5b89644ffd56d77b0f6472d8cd4d843601811b0adc71eb3f8f8203c812dbca7a154c061a9a3a9608dd3b8bc3faeea85d9bac2c0912351c34537129ddc667fa3430310a8918c77f72b03a91110f9f8dde0e9ba962f85e78ca9146d94bcf25fc9d26d28787daf4daf3d5f0010b14b7be84a584de43d53b3092a0f2fb8a32894d120e05a9edfe89228e1796e7d405d1730f00597be0c6da789d400754eb0d348316842fcb111deaf516dfc1334cba0a126d26350992d5a515ed95388de27046f66fce32343a012a04b6c5c1ea0086a5b8a5ef2fdbcdc7d3a2206007c37a69e99914902f3555d2e18d1a6cf5c96f8a199796cf8cc92080a0f9b9ae12d0eebc34fd03fb9281855ca2ed662954b4058e550dab8dbe20377f9fb24c1fee31d81c743db2b373da115e4424ceeb19271c3a6a2e7b1dd3f6ff2ac1f400ca6fe5476718a8b2ed0f45b78ca8a5cf859bb0752697e1dddc724a7ef07bbe3611b7bbf11c67c64850899b28d5e62a9584b503ba28cf33cfa8874dd35788d86a7b5659eddf67cd0ccd3bde729a3efc4b573ec17592e96bfd10c6a195b69e91d2bde5c197461c276c3cf21bad5da62a5d8aaa273a195e80d988ee1c508d305e7786994d09a0d6f10a690370a47fecbec2bc2308b937ec666dffbab76f4b4a21638dff8906cd361530eb3e32bf095ed4b499669d9263d4cb552bef4538ae26a790f4f0e58c3b834172df96f4aa25fe00fb0370165c6c8c372c6a722b67a42fe0292ddbe47463144a9e9be3b2670c6f11c13fff6882282a750ee4fb3b6c20c05a9e1695f56753167e3cbca97b479c427ad04a976fac8e21c748d94e3240ffef0eabcdbcba898bbd08884c5eb5d7f9d33e4111fe05f9dbf25c8b6bf8b6dee3f33e660943e6511eb519a34d86910fa76d274c3b6f49f4d7eebbd963ed1639503390fd37a087b394603235aac6f4a8b2e80320522a1e4c5c6815102b8125ed235de64486c25394b7953d009d0992762d9e7db78e8052f1e41c4788c938ea9aec84045ca9279909f02f268bf15250abc89429dc59652ef33ef13b60eb140c10f8e666d9dfdcd43d6ca64c9a48a272d79f6fc4e71be5125eaadded1818b28ea8431cf83a4cf7433533a6de195c59e587ca3f56877b62a0050dcbdea069bc462591a5d9e19c2e5946af0c8e5e87cd44011b999bce703cf11d3fd2e0e41d83d77f23eb17f2e248f144b2aa24089d61dafc8888d48866cb92af89014bc3cd95661516c389161b5f1a7dda17dbb1db0fcd05d6512cf2fd2bb1a2636234ada25bd1374a9fcef7d6c67dda194aac9d4a1b906d4a05b985e289d78481cbb43dfacfe4be66c01058e900f08001b9b5697bd283ed25f090d2fbc162c39e53b935494b3038a370145d6a69afd86231c6371d2eecd7504b5468204df7df593a14d3e3fa4eb8dbd6be9e18ba98163dcb2776e9e0bbb1e55a038c4229f586aed938215ad264c8f6fb35fd18188adfe99ebd2bf433bf019dd85573368a2470aa1fb84816b8113a024d1a4864b059c652a4658d65c529bbd529e6f9bdb44aa4fcefdfa28184ea90874734277d4dada764866ce6225d4fc63f22073cc39319890521cb015846c085f48e9354d9ba6cfb30758c1be393be1110cd67cbc82501861e3410f537f323d54f0a0afd26f96a39b029ba1d2ee5f3f90439fa42b8f025e9cb83c9c0fe27d38f5ad83b716ceea3053be5726aa67cfa06d9475a2cd08fa0cd07808352e999e36cd663d6c66ad406c9040b878e36bf5d1303b3efc42931139d379e7212034ce2cc327d3a6ea923a73eb3e2b11d369e0f9da8f65173cc36b9644add703b8d2642a6d47e962c794d4cc8274a840cc147f2191b41e9c85ca11810c6bb3c4dfb1e03fdeb101996d8cd79e182e47d3ea225199a2eee62eb3b90c75db71faeb283ca3eaedeac7fba5b08b858e05dfbecb20ce5bc02be0709baf18d8815bd10a8909e4c7a41ef3246b81921deb2dbcde2e589d493b89edacfd471c06beb86a8153c5a6606adb48cd145f1e37fffa3217fe729f1ae77445df445367a01979d8197b6e2b6adb6713d6a1cb077e20a825f89da00cf5efa364b7230d8fc7415d5eab427aecca33316d4072dafbe6b62f261dd0a1f6c7161b51e7f8d784092d06dabdd6be343706281e51d59a8951f114391978e2db3ab5797a47f89fa39a0e05f6863b81f82ce80cf8bd6011e321c875a432fa17dcfe884bf39705434add274cd8969a1c8c7ceaa6e0d2460299a26ee01287b55335 +erase 0,1 +erase 0,3 +erase 1,2 +erase 1,3 +case 4 2 16 zero +parity 0 00000000000000000000000000000000 +parity 1 00000000000000000000000000000000 +erase 0,1 +erase 0,4 +erase 2,3 +erase 2,4 +case 16 4 16 ff +parity 0 ffffffffffffffffffffffffffffffff +parity 1 ffffffffffffffffffffffffffffffff +parity 2 ffffffffffffffffffffffffffffffff +parity 3 ffffffffffffffffffffffffffffffff +erase 0,1,2,3 +erase 0,8,16,17 +erase 8,16,17,18 +erase 12,13,14,15 +case 1 1 1 lcg +parity 0 87 +erase 0 +case 1 1 9 lcg +parity 0 8763c23ca4b05c1d0a +erase 0 +case 2 1 5 lcg +parity 0 cd8575a93e +erase 0 +erase 1 +case 255 1 1 lcg +parity 0 51 +erase 0 +erase 127 +erase 254 +case 255 1 3 lcg +parity 0 512073 +erase 0 +erase 127 +erase 254 +case 1 255 1 lcg +parity 0 0d +parity 1 0d +parity 2 0d +parity 3 0d +parity 4 0d +parity 5 0d +parity 6 0d +parity 7 0d +parity 8 0d +parity 9 0d +parity 10 0d +parity 11 0d +parity 12 0d +parity 13 0d +parity 14 0d +parity 15 0d +parity 16 0d +parity 17 0d +parity 18 0d +parity 19 0d +parity 20 0d +parity 21 0d +parity 22 0d +parity 23 0d +parity 24 0d +parity 25 0d +parity 26 0d +parity 27 0d +parity 28 0d +parity 29 0d +parity 30 0d +parity 31 0d +parity 32 0d +parity 33 0d +parity 34 0d +parity 35 0d +parity 36 0d +parity 37 0d +parity 38 0d +parity 39 0d +parity 40 0d +parity 41 0d +parity 42 0d +parity 43 0d +parity 44 0d +parity 45 0d +parity 46 0d +parity 47 0d +parity 48 0d +parity 49 0d +parity 50 0d +parity 51 0d +parity 52 0d +parity 53 0d +parity 54 0d +parity 55 0d +parity 56 0d +parity 57 0d +parity 58 0d +parity 59 0d +parity 60 0d +parity 61 0d +parity 62 0d +parity 63 0d +parity 64 0d +parity 65 0d +parity 66 0d +parity 67 0d +parity 68 0d +parity 69 0d +parity 70 0d +parity 71 0d +parity 72 0d +parity 73 0d +parity 74 0d +parity 75 0d +parity 76 0d +parity 77 0d +parity 78 0d +parity 79 0d +parity 80 0d +parity 81 0d +parity 82 0d +parity 83 0d +parity 84 0d +parity 85 0d +parity 86 0d +parity 87 0d +parity 88 0d +parity 89 0d +parity 90 0d +parity 91 0d +parity 92 0d +parity 93 0d +parity 94 0d +parity 95 0d +parity 96 0d +parity 97 0d +parity 98 0d +parity 99 0d +parity 100 0d +parity 101 0d +parity 102 0d +parity 103 0d +parity 104 0d +parity 105 0d +parity 106 0d +parity 107 0d +parity 108 0d +parity 109 0d +parity 110 0d +parity 111 0d +parity 112 0d +parity 113 0d +parity 114 0d +parity 115 0d +parity 116 0d +parity 117 0d +parity 118 0d +parity 119 0d +parity 120 0d +parity 121 0d +parity 122 0d +parity 123 0d +parity 124 0d +parity 125 0d +parity 126 0d +parity 127 0d +parity 128 0d +parity 129 0d +parity 130 0d +parity 131 0d +parity 132 0d +parity 133 0d +parity 134 0d +parity 135 0d +parity 136 0d +parity 137 0d +parity 138 0d +parity 139 0d +parity 140 0d +parity 141 0d +parity 142 0d +parity 143 0d +parity 144 0d +parity 145 0d +parity 146 0d +parity 147 0d +parity 148 0d +parity 149 0d +parity 150 0d +parity 151 0d +parity 152 0d +parity 153 0d +parity 154 0d +parity 155 0d +parity 156 0d +parity 157 0d +parity 158 0d +parity 159 0d +parity 160 0d +parity 161 0d +parity 162 0d +parity 163 0d +parity 164 0d +parity 165 0d +parity 166 0d +parity 167 0d +parity 168 0d +parity 169 0d +parity 170 0d +parity 171 0d +parity 172 0d +parity 173 0d +parity 174 0d +parity 175 0d +parity 176 0d +parity 177 0d +parity 178 0d +parity 179 0d +parity 180 0d +parity 181 0d +parity 182 0d +parity 183 0d +parity 184 0d +parity 185 0d +parity 186 0d +parity 187 0d +parity 188 0d +parity 189 0d +parity 190 0d +parity 191 0d +parity 192 0d +parity 193 0d +parity 194 0d +parity 195 0d +parity 196 0d +parity 197 0d +parity 198 0d +parity 199 0d +parity 200 0d +parity 201 0d +parity 202 0d +parity 203 0d +parity 204 0d +parity 205 0d +parity 206 0d +parity 207 0d +parity 208 0d +parity 209 0d +parity 210 0d +parity 211 0d +parity 212 0d +parity 213 0d +parity 214 0d +parity 215 0d +parity 216 0d +parity 217 0d +parity 218 0d +parity 219 0d +parity 220 0d +parity 221 0d +parity 222 0d +parity 223 0d +parity 224 0d +parity 225 0d +parity 226 0d +parity 227 0d +parity 228 0d +parity 229 0d +parity 230 0d +parity 231 0d +parity 232 0d +parity 233 0d +parity 234 0d +parity 235 0d +parity 236 0d +parity 237 0d +parity 238 0d +parity 239 0d +parity 240 0d +parity 241 0d +parity 242 0d +parity 243 0d +parity 244 0d +parity 245 0d +parity 246 0d +parity 247 0d +parity 248 0d +parity 249 0d +parity 250 0d +parity 251 0d +parity 252 0d +parity 253 0d +parity 254 0d +erase 0 +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +case 1 255 4 lcg +parity 0 0d2f7919 +parity 1 0d2f7919 +parity 2 0d2f7919 +parity 3 0d2f7919 +parity 4 0d2f7919 +parity 5 0d2f7919 +parity 6 0d2f7919 +parity 7 0d2f7919 +parity 8 0d2f7919 +parity 9 0d2f7919 +parity 10 0d2f7919 +parity 11 0d2f7919 +parity 12 0d2f7919 +parity 13 0d2f7919 +parity 14 0d2f7919 +parity 15 0d2f7919 +parity 16 0d2f7919 +parity 17 0d2f7919 +parity 18 0d2f7919 +parity 19 0d2f7919 +parity 20 0d2f7919 +parity 21 0d2f7919 +parity 22 0d2f7919 +parity 23 0d2f7919 +parity 24 0d2f7919 +parity 25 0d2f7919 +parity 26 0d2f7919 +parity 27 0d2f7919 +parity 28 0d2f7919 +parity 29 0d2f7919 +parity 30 0d2f7919 +parity 31 0d2f7919 +parity 32 0d2f7919 +parity 33 0d2f7919 +parity 34 0d2f7919 +parity 35 0d2f7919 +parity 36 0d2f7919 +parity 37 0d2f7919 +parity 38 0d2f7919 +parity 39 0d2f7919 +parity 40 0d2f7919 +parity 41 0d2f7919 +parity 42 0d2f7919 +parity 43 0d2f7919 +parity 44 0d2f7919 +parity 45 0d2f7919 +parity 46 0d2f7919 +parity 47 0d2f7919 +parity 48 0d2f7919 +parity 49 0d2f7919 +parity 50 0d2f7919 +parity 51 0d2f7919 +parity 52 0d2f7919 +parity 53 0d2f7919 +parity 54 0d2f7919 +parity 55 0d2f7919 +parity 56 0d2f7919 +parity 57 0d2f7919 +parity 58 0d2f7919 +parity 59 0d2f7919 +parity 60 0d2f7919 +parity 61 0d2f7919 +parity 62 0d2f7919 +parity 63 0d2f7919 +parity 64 0d2f7919 +parity 65 0d2f7919 +parity 66 0d2f7919 +parity 67 0d2f7919 +parity 68 0d2f7919 +parity 69 0d2f7919 +parity 70 0d2f7919 +parity 71 0d2f7919 +parity 72 0d2f7919 +parity 73 0d2f7919 +parity 74 0d2f7919 +parity 75 0d2f7919 +parity 76 0d2f7919 +parity 77 0d2f7919 +parity 78 0d2f7919 +parity 79 0d2f7919 +parity 80 0d2f7919 +parity 81 0d2f7919 +parity 82 0d2f7919 +parity 83 0d2f7919 +parity 84 0d2f7919 +parity 85 0d2f7919 +parity 86 0d2f7919 +parity 87 0d2f7919 +parity 88 0d2f7919 +parity 89 0d2f7919 +parity 90 0d2f7919 +parity 91 0d2f7919 +parity 92 0d2f7919 +parity 93 0d2f7919 +parity 94 0d2f7919 +parity 95 0d2f7919 +parity 96 0d2f7919 +parity 97 0d2f7919 +parity 98 0d2f7919 +parity 99 0d2f7919 +parity 100 0d2f7919 +parity 101 0d2f7919 +parity 102 0d2f7919 +parity 103 0d2f7919 +parity 104 0d2f7919 +parity 105 0d2f7919 +parity 106 0d2f7919 +parity 107 0d2f7919 +parity 108 0d2f7919 +parity 109 0d2f7919 +parity 110 0d2f7919 +parity 111 0d2f7919 +parity 112 0d2f7919 +parity 113 0d2f7919 +parity 114 0d2f7919 +parity 115 0d2f7919 +parity 116 0d2f7919 +parity 117 0d2f7919 +parity 118 0d2f7919 +parity 119 0d2f7919 +parity 120 0d2f7919 +parity 121 0d2f7919 +parity 122 0d2f7919 +parity 123 0d2f7919 +parity 124 0d2f7919 +parity 125 0d2f7919 +parity 126 0d2f7919 +parity 127 0d2f7919 +parity 128 0d2f7919 +parity 129 0d2f7919 +parity 130 0d2f7919 +parity 131 0d2f7919 +parity 132 0d2f7919 +parity 133 0d2f7919 +parity 134 0d2f7919 +parity 135 0d2f7919 +parity 136 0d2f7919 +parity 137 0d2f7919 +parity 138 0d2f7919 +parity 139 0d2f7919 +parity 140 0d2f7919 +parity 141 0d2f7919 +parity 142 0d2f7919 +parity 143 0d2f7919 +parity 144 0d2f7919 +parity 145 0d2f7919 +parity 146 0d2f7919 +parity 147 0d2f7919 +parity 148 0d2f7919 +parity 149 0d2f7919 +parity 150 0d2f7919 +parity 151 0d2f7919 +parity 152 0d2f7919 +parity 153 0d2f7919 +parity 154 0d2f7919 +parity 155 0d2f7919 +parity 156 0d2f7919 +parity 157 0d2f7919 +parity 158 0d2f7919 +parity 159 0d2f7919 +parity 160 0d2f7919 +parity 161 0d2f7919 +parity 162 0d2f7919 +parity 163 0d2f7919 +parity 164 0d2f7919 +parity 165 0d2f7919 +parity 166 0d2f7919 +parity 167 0d2f7919 +parity 168 0d2f7919 +parity 169 0d2f7919 +parity 170 0d2f7919 +parity 171 0d2f7919 +parity 172 0d2f7919 +parity 173 0d2f7919 +parity 174 0d2f7919 +parity 175 0d2f7919 +parity 176 0d2f7919 +parity 177 0d2f7919 +parity 178 0d2f7919 +parity 179 0d2f7919 +parity 180 0d2f7919 +parity 181 0d2f7919 +parity 182 0d2f7919 +parity 183 0d2f7919 +parity 184 0d2f7919 +parity 185 0d2f7919 +parity 186 0d2f7919 +parity 187 0d2f7919 +parity 188 0d2f7919 +parity 189 0d2f7919 +parity 190 0d2f7919 +parity 191 0d2f7919 +parity 192 0d2f7919 +parity 193 0d2f7919 +parity 194 0d2f7919 +parity 195 0d2f7919 +parity 196 0d2f7919 +parity 197 0d2f7919 +parity 198 0d2f7919 +parity 199 0d2f7919 +parity 200 0d2f7919 +parity 201 0d2f7919 +parity 202 0d2f7919 +parity 203 0d2f7919 +parity 204 0d2f7919 +parity 205 0d2f7919 +parity 206 0d2f7919 +parity 207 0d2f7919 +parity 208 0d2f7919 +parity 209 0d2f7919 +parity 210 0d2f7919 +parity 211 0d2f7919 +parity 212 0d2f7919 +parity 213 0d2f7919 +parity 214 0d2f7919 +parity 215 0d2f7919 +parity 216 0d2f7919 +parity 217 0d2f7919 +parity 218 0d2f7919 +parity 219 0d2f7919 +parity 220 0d2f7919 +parity 221 0d2f7919 +parity 222 0d2f7919 +parity 223 0d2f7919 +parity 224 0d2f7919 +parity 225 0d2f7919 +parity 226 0d2f7919 +parity 227 0d2f7919 +parity 228 0d2f7919 +parity 229 0d2f7919 +parity 230 0d2f7919 +parity 231 0d2f7919 +parity 232 0d2f7919 +parity 233 0d2f7919 +parity 234 0d2f7919 +parity 235 0d2f7919 +parity 236 0d2f7919 +parity 237 0d2f7919 +parity 238 0d2f7919 +parity 239 0d2f7919 +parity 240 0d2f7919 +parity 241 0d2f7919 +parity 242 0d2f7919 +parity 243 0d2f7919 +parity 244 0d2f7919 +parity 245 0d2f7919 +parity 246 0d2f7919 +parity 247 0d2f7919 +parity 248 0d2f7919 +parity 249 0d2f7919 +parity 250 0d2f7919 +parity 251 0d2f7919 +parity 252 0d2f7919 +parity 253 0d2f7919 +parity 254 0d2f7919 +erase 0 +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +case 2 254 2 lcg +parity 0 1de3 +parity 1 061b +parity 2 47c9 +parity 3 5c31 +parity 4 7124 +parity 5 6adc +parity 6 f39d +parity 7 e865 +parity 8 c570 +parity 9 de88 +parity 10 9f5a +parity 11 84a2 +parity 12 a9b7 +parity 13 b24f +parity 14 8635 +parity 15 9dcd +parity 16 b0d8 +parity 17 ab20 +parity 18 eaf2 +parity 19 f10a +parity 20 dc1f +parity 21 c7e7 +parity 22 5ea6 +parity 23 455e +parity 24 684b +parity 25 73b3 +parity 26 3261 +parity 27 2999 +parity 28 048c +parity 29 1f74 +parity 30 6c78 +parity 31 7780 +parity 32 5a95 +parity 33 416d +parity 34 00bf +parity 35 1b47 +parity 36 3652 +parity 37 2daa +parity 38 b4eb +parity 39 af13 +parity 40 8206 +parity 41 99fe +parity 42 d82c +parity 43 c3d4 +parity 44 eec1 +parity 45 f539 +parity 46 c143 +parity 47 dabb +parity 48 f7ae +parity 49 ec56 +parity 50 ad84 +parity 51 b67c +parity 52 9b69 +parity 53 8091 +parity 54 19d0 +parity 55 0228 +parity 56 2f3d +parity 57 34c5 +parity 58 7517 +parity 59 6eef +parity 60 43fa +parity 61 5802 +parity 62 a5e2 +parity 63 be1a +parity 64 930f +parity 65 88f7 +parity 66 c925 +parity 67 d2dd +parity 68 ffc8 +parity 69 e430 +parity 70 7d71 +parity 71 6689 +parity 72 4b9c +parity 73 5064 +parity 74 11b6 +parity 75 0a4e +parity 76 275b +parity 77 3ca3 +parity 78 08d9 +parity 79 1321 +parity 80 3e34 +parity 81 25cc +parity 82 641e +parity 83 7fe6 +parity 84 52f3 +parity 85 490b +parity 86 d04a +parity 87 cbb2 +parity 88 e6a7 +parity 89 fd5f +parity 90 bc8d +parity 91 a775 +parity 92 8a60 +parity 93 9198 +parity 94 e294 +parity 95 f96c +parity 96 d479 +parity 97 cf81 +parity 98 8e53 +parity 99 95ab +parity 100 b8be +parity 101 a346 +parity 102 3a07 +parity 103 21ff +parity 104 0cea +parity 105 1712 +parity 106 56c0 +parity 107 4d38 +parity 108 602d +parity 109 7bd5 +parity 110 4faf +parity 111 5457 +parity 112 7942 +parity 113 62ba +parity 114 2368 +parity 115 3890 +parity 116 1585 +parity 117 0e7d +parity 118 973c +parity 119 8cc4 +parity 120 a1d1 +parity 121 ba29 +parity 122 fbfb +parity 123 e003 +parity 124 cd16 +parity 125 d6ee +parity 126 2acb +parity 127 3133 +parity 128 1c26 +parity 129 07de +parity 130 460c +parity 131 5df4 +parity 132 70e1 +parity 133 6b19 +parity 134 f258 +parity 135 e9a0 +parity 136 c4b5 +parity 137 df4d +parity 138 9e9f +parity 139 8567 +parity 140 a872 +parity 141 b38a +parity 142 87f0 +parity 143 9c08 +parity 144 b11d +parity 145 aae5 +parity 146 eb37 +parity 147 f0cf +parity 148 ddda +parity 149 c622 +parity 150 5f63 +parity 151 449b +parity 152 698e +parity 153 7276 +parity 154 33a4 +parity 155 285c +parity 156 0549 +parity 157 1eb1 +parity 158 6dbd +parity 159 7645 +parity 160 5b50 +parity 161 40a8 +parity 162 017a +parity 163 1a82 +parity 164 3797 +parity 165 2c6f +parity 166 b52e +parity 167 aed6 +parity 168 83c3 +parity 169 983b +parity 170 d9e9 +parity 171 c211 +parity 172 ef04 +parity 173 f4fc +parity 174 c086 +parity 175 db7e +parity 176 f66b +parity 177 ed93 +parity 178 ac41 +parity 179 b7b9 +parity 180 9aac +parity 181 8154 +parity 182 1815 +parity 183 03ed +parity 184 2ef8 +parity 185 3500 +parity 186 74d2 +parity 187 6f2a +parity 188 423f +parity 189 59c7 +parity 190 a427 +parity 191 bfdf +parity 192 92ca +parity 193 8932 +parity 194 c8e0 +parity 195 d318 +parity 196 fe0d +parity 197 e5f5 +parity 198 7cb4 +parity 199 674c +parity 200 4a59 +parity 201 51a1 +parity 202 1073 +parity 203 0b8b +parity 204 269e +parity 205 3d66 +parity 206 091c +parity 207 12e4 +parity 208 3ff1 +parity 209 2409 +parity 210 65db +parity 211 7e23 +parity 212 5336 +parity 213 48ce +parity 214 d18f +parity 215 ca77 +parity 216 e762 +parity 217 fc9a +parity 218 bd48 +parity 219 a6b0 +parity 220 8ba5 +parity 221 905d +parity 222 e351 +parity 223 f8a9 +parity 224 d5bc +parity 225 ce44 +parity 226 8f96 +parity 227 946e +parity 228 b97b +parity 229 a283 +parity 230 3bc2 +parity 231 203a +parity 232 0d2f +parity 233 16d7 +parity 234 5705 +parity 235 4cfd +parity 236 61e8 +parity 237 7a10 +parity 238 4e6a +parity 239 5592 +parity 240 7887 +parity 241 637f +parity 242 22ad +parity 243 3955 +parity 244 1440 +parity 245 0fb8 +parity 246 96f9 +parity 247 8d01 +parity 248 a014 +parity 249 bbec +parity 250 fa3e +parity 251 e1c6 +parity 252 ccd3 +parity 253 d72b +erase 0,1 +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253 +erase 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +case 128 128 1 lcg +parity 0 3f +parity 1 67 +parity 2 58 +parity 3 ac +parity 4 d6 +parity 5 45 +parity 6 7a +parity 7 81 +parity 8 b0 +parity 9 5d +parity 10 5a +parity 11 e7 +parity 12 35 +parity 13 da +parity 14 5d +parity 15 31 +parity 16 03 +parity 17 35 +parity 18 5c +parity 19 a1 +parity 20 00 +parity 21 4a +parity 22 70 +parity 23 5b +parity 24 13 +parity 25 b4 +parity 26 9c +parity 27 dc +parity 28 f6 +parity 29 9b +parity 30 f1 +parity 31 38 +parity 32 20 +parity 33 59 +parity 34 f9 +parity 35 84 +parity 36 dc +parity 37 4d +parity 38 a6 +parity 39 68 +parity 40 e8 +parity 41 2c +parity 42 8b +parity 43 93 +parity 44 e2 +parity 45 75 +parity 46 55 +parity 47 d1 +parity 48 43 +parity 49 0c +parity 50 9f +parity 51 c7 +parity 52 c0 +parity 53 cc +parity 54 89 +parity 55 77 +parity 56 68 +parity 57 d4 +parity 58 56 +parity 59 a5 +parity 60 b3 +parity 61 c1 +parity 62 12 +parity 63 e4 +parity 64 0d +parity 65 a8 +parity 66 88 +parity 67 9c +parity 68 f9 +parity 69 b9 +parity 70 27 +parity 71 b9 +parity 72 c3 +parity 73 3a +parity 74 84 +parity 75 f9 +parity 76 d9 +parity 77 0a +parity 78 8c +parity 79 79 +parity 80 c7 +parity 81 f4 +parity 82 46 +parity 83 ff +parity 84 ce +parity 85 e7 +parity 86 1c +parity 87 72 +parity 88 07 +parity 89 c2 +parity 90 d1 +parity 91 89 +parity 92 7c +parity 93 ab +parity 94 8d +parity 95 80 +parity 96 3c +parity 97 aa +parity 98 b7 +parity 99 ee +parity 100 5c +parity 101 14 +parity 102 98 +parity 103 05 +parity 104 73 +parity 105 07 +parity 106 9a +parity 107 01 +parity 108 0d +parity 109 f3 +parity 110 9f +parity 111 da +parity 112 3e +parity 113 d9 +parity 114 67 +parity 115 d9 +parity 116 69 +parity 117 7f +parity 118 e2 +parity 119 76 +parity 120 71 +parity 121 a5 +parity 122 7a +parity 123 8d +parity 124 cc +parity 125 5b +parity 126 2f +parity 127 ba +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127 +erase 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191 +erase 64,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +case 128 128 4 lcg +parity 0 3fd0cd4e +parity 1 67acce3d +parity 2 58c5aec7 +parity 3 ac1f402d +parity 4 d680e2ae +parity 5 454d646d +parity 6 7a15f048 +parity 7 81db42c0 +parity 8 b041c75a +parity 9 5dba99ff +parity 10 5a4d1ce6 +parity 11 e7ca523e +parity 12 35ee7d2a +parity 13 da241b58 +parity 14 5de4a50e +parity 15 31caf2ad +parity 16 0369da6c +parity 17 35997fc0 +parity 18 5c0294e7 +parity 19 a1971585 +parity 20 002a2321 +parity 21 4a974d65 +parity 22 70d4ab7a +parity 23 5b1a8ec6 +parity 24 1395171f +parity 25 b4a002a7 +parity 26 9cf90bbe +parity 27 dcdf3f10 +parity 28 f6cf46fa +parity 29 9b9a0b6a +parity 30 f11027db +parity 31 3833f7af +parity 32 20828a5b +parity 33 596d355f +parity 34 f9f209d1 +parity 35 84b632c0 +parity 36 dce8d468 +parity 37 4d726181 +parity 38 a6835e72 +parity 39 68910de5 +parity 40 e80d1978 +parity 41 2c63092a +parity 42 8bc522ee +parity 43 930f5c98 +parity 44 e221177a +parity 45 75147f7c +parity 46 558b8934 +parity 47 d1a202c7 +parity 48 4311487e +parity 49 0c846b66 +parity 50 9f5d19d2 +parity 51 c761f90c +parity 52 c07a9e70 +parity 53 ccfcf9ab +parity 54 89c8a8fc +parity 55 7779d864 +parity 56 685c954e +parity 57 d448cdf2 +parity 58 56421c35 +parity 59 a59f2518 +parity 60 b3e1bb66 +parity 61 c157d8f2 +parity 62 122452aa +parity 63 e4037089 +parity 64 0dd99b70 +parity 65 a8a9472c +parity 66 8834af03 +parity 67 9c61990b +parity 68 f9545ded +parity 69 b90d3aa6 +parity 70 27bfc30d +parity 71 b92302fc +parity 72 c3cba2dd +parity 73 3acc4530 +parity 74 84d610c8 +parity 75 f9fac232 +parity 76 d97bc887 +parity 77 0a27b288 +parity 78 8c8d80a1 +parity 79 79cd4482 +parity 80 c72e0490 +parity 81 f42d6c27 +parity 82 46173b48 +parity 83 ff95a4c6 +parity 84 ceb1fbfa +parity 85 e7e8039e +parity 86 1cc326f0 +parity 87 725f2675 +parity 88 07c0d0bf +parity 89 c2b987f5 +parity 90 d1c73bfb +parity 91 89e3d746 +parity 92 7c66fdde +parity 93 ab32e989 +parity 94 8dbc7edf +parity 95 8093750b +parity 96 3c645f96 +parity 97 aa542e61 +parity 98 b77c83f1 +parity 99 eeb2e7f6 +parity 100 5c0c86c6 +parity 101 14d3453b +parity 102 98ce7f22 +parity 103 05d8e6a8 +parity 104 73717e05 +parity 105 070cb0d5 +parity 106 9a78d5aa +parity 107 01c97566 +parity 108 0d924c1d +parity 109 f34be814 +parity 110 9f02b477 +parity 111 da85441b +parity 112 3e111d43 +parity 113 d9b57355 +parity 114 673936e5 +parity 115 d90c4645 +parity 116 691c9e27 +parity 117 7fd06f61 +parity 118 e21706a9 +parity 119 76758e4c +parity 120 719d6a94 +parity 121 a5950a27 +parity 122 7a0b1ba9 +parity 123 8d76695b +parity 124 cca2e422 +parity 125 5b805cd2 +parity 126 2fe56bae +parity 127 baf031ad +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127 +erase 0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191 +erase 64,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +case 200 56 2 lcg +parity 0 946f +parity 1 3f89 +parity 2 2f17 +parity 3 2700 +parity 4 5d92 +parity 5 ba5f +parity 6 95aa +parity 7 3395 +parity 8 2109 +parity 9 a097 +parity 10 2313 +parity 11 55e1 +parity 12 9e65 +parity 13 53ce +parity 14 ce44 +parity 15 c58b +parity 16 f2bc +parity 17 64e3 +parity 18 5677 +parity 19 ba5f +parity 20 d1c9 +parity 21 e3e9 +parity 22 7b7b +parity 23 9714 +parity 24 215d +parity 25 f4c8 +parity 26 4eba +parity 27 4e9d +parity 28 2af2 +parity 29 59bb +parity 30 9674 +parity 31 8e33 +parity 32 334b +parity 33 a29b +parity 34 21eb +parity 35 6ba9 +parity 36 7b98 +parity 37 1ada +parity 38 67f0 +parity 39 70b9 +parity 40 19a7 +parity 41 ac6b +parity 42 04b4 +parity 43 1bdb +parity 44 fd80 +parity 45 91c6 +parity 46 b69e +parity 47 da95 +parity 48 1524 +parity 49 4232 +parity 50 8464 +parity 51 f86e +parity 52 2a39 +parity 53 6429 +parity 54 4bcf +parity 55 ea24 +erase 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55 +erase 0,7,14,21,28,35,42,50,57,64,71,78,85,92,100,107,114,121,128,135,142,150,157,164,171,178,185,192,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227 +erase 100,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254 +erase 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199 +case 250 6 5 lcg +parity 0 cd4bb59e20 +parity 1 15f9bc891f +parity 2 8fb793679f +parity 3 fbf644d8e5 +parity 4 009c7d60c8 +parity 5 4cd67a51d4 +erase 0,1,2,3,4,5 +erase 0,83,166,250,251,252 +erase 125,250,251,252,253,254 +erase 244,245,246,247,248,249 diff --git a/packages/cultnet-rs/tests/media_fec.rs b/packages/cultnet-rs/tests/media_fec.rs index 4af9f243..c76709d4 100644 --- a/packages/cultnet-rs/tests/media_fec.rs +++ b/packages/cultnet-rs/tests/media_fec.rs @@ -1141,3 +1141,198 @@ fn a_budget_at_the_media_record_ceiling_admits_large_shards_end_to_end() { .unwrap(); decode_media_wire_record(&wire).unwrap(); } + +// --------------------------------------------------------------------------- +// Known answers from the fixture: every standard geometry and the policy's +// edges, byte for byte. The fixture's header states how its data is made. +// --------------------------------------------------------------------------- + +/// The data of a fixture case, as the fixture's header defines it. +fn kat_data(k: usize, m: usize, shard_bytes: usize, fill: &str) -> Vec> { + (0..k) + .map(|i| match fill { + "lcg" => bytes((k * 65_536 + m * 256 + i) as u32, shard_bytes), + "zero" => vec![0; shard_bytes], + "ff" => vec![0xff; shard_bytes], + other => panic!("unknown fill {other}"), + }) + .collect() +} + +struct KatCase { + k: usize, + m: usize, + shard_bytes: usize, + fill: String, + parity: Vec>, + erasures: Vec>, +} + +impl KatCase { + fn name(&self) -> String { + format!("k={} m={} {} bytes {}", self.k, self.m, self.shard_bytes, self.fill) + } + + /// The case's data as an audio block: the codec path that takes any `(k, m)`. + fn packets(&self) -> Vec { + let mut packets = audio_run(self.k as u64, self.shard_bytes); + for (packet, data) in packets.iter_mut().zip(kat_data(self.k, self.m, self.shard_bytes, &self.fill)) { + packet.payload = data; + } + packets + } + + fn policy(&self) -> MediaFecPolicy { + MediaFecPolicy { + audio_data_shards: self.k as u16, + audio_parity_shards: self.m as u16, + max_wire_bytes: GAMECULT_MEDIA_MAX_WIRE_BYTES, + ..STANDARD + } + } +} + +fn unhex(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text[at..at + 2], 16).expect("fixture hex")) + .collect() +} + +fn kat_cases() -> Vec { + let mut cases: Vec = Vec::new(); + for line in include_str!("fixtures/media_fec_rs_gf256_v1.kat").lines() { + let fields: Vec<&str> = line.split_whitespace().collect(); + match fields.as_slice() { + [] => {} + [comment, ..] if comment.starts_with('#') => {} + ["case", k, m, shard_bytes, fill] => cases.push(KatCase { + k: k.parse().unwrap(), + m: m.parse().unwrap(), + shard_bytes: shard_bytes.parse().unwrap(), + fill: fill.to_string(), + parity: Vec::new(), + erasures: Vec::new(), + }), + ["parity", index, bytes] => { + let case = cases.last_mut().expect("parity follows a case"); + assert_eq!(index.parse::().unwrap(), case.parity.len(), "{}", case.name()); + case.parity.push(unhex(bytes)); + } + ["erase", slots] => cases + .last_mut() + .expect("erase follows a case") + .erasures + .push(slots.split(',').map(|slot| slot.parse().unwrap()).collect()), + other => panic!("unreadable fixture line {other:?}"), + } + } + cases +} + +/// Parity bytes of every fixture case, byte for byte. If these ever change, +/// `rs-gf256-v1` changed meaning: bump the scheme id, do not edit the fixture. +#[test] +fn every_rs_gf256_v1_known_answer_parity_matches_byte_for_byte() { + let cases = kat_cases(); + assert_eq!(cases.len(), 35, "the fixture's case count"); + for case in &cases { + assert_eq!(case.parity.len(), case.m, "{}", case.name()); + let parity = protect_audio_block(&case.packets(), &case.policy(), provenance()) + .unwrap_or_else(|error| panic!("{}: {error}", case.name())); + let produced: Vec = parity.iter().map(|shard| hex(&shard.payload)).collect(); + let expected: Vec = case.parity.iter().map(|shard| hex(shard)).collect(); + assert_eq!(produced, expected, "{}", case.name()); + } +} + +/// Every fixture erasure pattern, data and parity mixed, recovers exactly +/// the erased data from the fixture's own parity bytes. +#[test] +fn every_rs_gf256_v1_known_erasure_pattern_recovers_the_data() { + for case in kat_cases() { + let packets = case.packets(); + let mut parity = protect_audio_block(&packets, &case.policy(), provenance()).unwrap(); + for (shard, bytes) in parity.iter_mut().zip(&case.parity) { + shard.payload = bytes.clone(); // decode from the fixture, not from the encoder + } + assert!(!case.erasures.is_empty(), "{}", case.name()); + for erased in &case.erasures { + let data: Vec<_> = + (0..case.k).filter(|slot| !erased.contains(slot)).map(|slot| packets[slot].clone()).collect(); + let present: Vec<_> = (0..case.m) + .filter(|slot| !erased.contains(&(case.k + slot))) + .map(|slot| parity[slot].clone()) + .collect(); + let recovered = recover_audio_block(&present, &data) + .unwrap_or_else(|error| panic!("{} erased {erased:?}: {error}", case.name())); + let lost: Vec = erased.iter().copied().filter(|&slot| slot < case.k).collect(); + assert_eq!(recovered.len(), lost.len(), "{} erased {erased:?}", case.name()); + for (packet, slot) in recovered.iter().zip(lost) { + assert_eq!(packet.packet_id, packets[slot].packet_id, "{} erased {erased:?}", case.name()); + assert_eq!(packet.payload, packets[slot].payload, "{} erased {erased:?}", case.name()); + } + } + } +} + +/// Seeded random geometries past the fixture's: any `e <= m` erasures, data +/// and parity mixed, recover the data exactly, and `m + 1` are refused. +#[test] +fn random_erasures_up_to_m_recover_exactly_and_m_plus_one_is_refused() { + let mut state = 0x9e37_79b9_u32; + let mut next = |bound: usize| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as usize % bound + }; + for trial in 0..300 { + let k = 1 + next(48); + let m = 1 + next(24); + let shard_bytes = 1 + next(40); + let mut packets = audio_run(k as u64, shard_bytes); + for (index, packet) in packets.iter_mut().enumerate() { + packet.payload = bytes((trial * 1_000 + index) as u32, shard_bytes); + } + let policy = MediaFecPolicy { + audio_data_shards: k as u16, + audio_parity_shards: m as u16, + ..STANDARD + }; + let parity = protect_audio_block(&packets, &policy, provenance()).unwrap(); + for erasures in [1 + next(m), m + 1] { + if erasures + 1 > k + m { + continue; // at least one parity shard must arrive to recover with + } + // A random erasure set that leaves at least one parity shard. + let mut slots: Vec = (0..k + m).collect(); + let erased = loop { + for at in 0..slots.len() { + let swap = at + next(slots.len() - at); + slots.swap(at, swap); + } + let erased = slots[..erasures].to_vec(); + if (k..k + m).any(|slot| !erased.contains(&slot)) { + break erased; + } + }; + let data: Vec<_> = + (0..k).filter(|slot| !erased.contains(slot)).map(|slot| packets[slot].clone()).collect(); + let present: Vec<_> = + (0..m).filter(|slot| !erased.contains(&(k + slot))).map(|slot| parity[slot].clone()).collect(); + let outcome = recover_audio_block(&present, &data); + let context = format!("trial {trial}: k={k} m={m} {shard_bytes} bytes erased {erased:?}"); + if erasures <= m { + let mut all = data; + all.extend(outcome.unwrap_or_else(|error| panic!("{context}: {error}"))); + all.sort_by_key(|packet| packet.packet_id); + let payloads: Vec<_> = all.iter().map(|packet| &packet.payload).collect(); + let expected: Vec<_> = packets.iter().map(|packet| &packet.payload).collect(); + assert_eq!(payloads, expected, "{context}"); + } else { + assert!(matches!(outcome, Err(MediaFecError::BeyondRepair { .. })), "{context}: {outcome:?}"); + } + } + } +} diff --git a/packages/cultnet-rs/tests/media_fec_bench.rs b/packages/cultnet-rs/tests/media_fec_bench.rs new file mode 100644 index 00000000..40c9de78 --- /dev/null +++ b/packages/cultnet-rs/tests/media_fec_bench.rs @@ -0,0 +1,88 @@ +//! Timings for the media FEC codec, without a bench framework. +//! +//! Ignored by default; run in release, where the numbers mean something: +//! +//! ```text +//! cargo test --release --test media_fec_bench -- --ignored --nocapture +//! ``` +//! +//! Each line is nanoseconds per call through the public surface, so record +//! validation and cloning are included. The geometries are the ones the codec +//! meets: standard audio, the widest standard video block at a datagram, and +//! the worst recovery the policy admits (255 + 1). + +use std::hint::black_box; +use std::time::Instant; + +use cultnet_rs::{ + GAMECULT_MEDIA_MAX_WIRE_BYTES, GameCultMediaAudioPacketRecord, MediaFecPolicy, + MediaWireProvenance, protect_audio_block, recover_audio_block, +}; + +fn provenance() -> MediaWireProvenance<'static> { + MediaWireProvenance { + stored_at: "unix:1700000000000", + runtime_id: "bench", + role: "bench.media", + producer: "bench", + } +} + +fn packets(k: usize, len: usize) -> Vec { + let mut state = 0x2545_f491_u32; + (0..k as u64) + .map(|index| GameCultMediaAudioPacketRecord { + stream_id: "bench".to_string(), + session_id: "bench".to_string(), + packet_id: index, + codec: "opus".to_string(), + pts_ticks: 960 * index as i64, + duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 960 * index as i64 + 5_000, + payload: (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(), + }) + .collect() +} + +/// Mean nanoseconds per call over at least `min_ms`, after three warm-up calls. +fn time(mut f: impl FnMut(), min_ms: u128) -> f64 { + for _ in 0..3 { + f(); + } + let start = Instant::now(); + let mut calls = 0_u64; + while start.elapsed().as_millis() < min_ms { + f(); + calls += 1; + } + start.elapsed().as_nanos() as f64 / calls as f64 +} + +#[test] +#[ignore = "timing, not a test: run in release with --ignored --nocapture"] +fn media_fec_timings() { + for (k, m, len) in [(4, 2, 160), (4, 2, 1300), (16, 4, 1200), (255, 1, 1300)] { + let policy = MediaFecPolicy { + audio_data_shards: k as u16, + audio_parity_shards: m as u16, + max_wire_bytes: GAMECULT_MEDIA_MAX_WIRE_BYTES, + ..MediaFecPolicy::STANDARD + }; + let block = packets(k, len); + let encode = time(|| drop(black_box(protect_audio_block(black_box(&block), &policy, provenance()))), 300); + let parity = protect_audio_block(&block, &policy, provenance()).unwrap(); + let lost = m.min(k); + let survivors = &block[lost..]; + let decode = time(|| drop(black_box(recover_audio_block(black_box(&parity), survivors))), 300); + println!("k={k:3} m={m:3} len={len:5}: protect {encode:12.0} ns | recover {lost} lost {decode:12.0} ns"); + } +} diff --git a/packages/cultnet-rs/tests/rudp_connect_generation.rs b/packages/cultnet-rs/tests/rudp_connect_generation.rs index b456162c..92f83322 100644 --- a/packages/cultnet-rs/tests/rudp_connect_generation.rs +++ b/packages/cultnet-rs/tests/rudp_connect_generation.rs @@ -575,9 +575,30 @@ fn a_flush_that_a_timeout_ended_names_the_timeout() -> Result<()> { // Hold-buffer boundaries: 1,024 frames and 64 channels are held; one more is // refused. +/// An acknowledgement that names only `sequence`. +fn ack_naming(sequence: u32) -> CultNetRudpPacket { + CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Ack, + connection_id: CONNECTION_ID, + sequence: 0, + ack: sequence, + ack_mask: 0, + channel_id: "control".into(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + } +} + /// A connected pair with one sequence the receiver will never see, so every -/// later ordered frame waits behind it. The sender's window is kept open by -/// acknowledging what the receiver has. +/// later ordered frame waits behind it. The sender's flow window would stop it +/// 1,023 sequences above its lowest unacked packet, so the missing one is +/// acknowledged by hand (the receiver never saw it) and the rest by what the +/// receiver has: the receiver's hold buffer is what this probes, not the sender. fn held_behind_a_gap( frames: usize, channel_of: impl Fn(usize) -> String, @@ -585,7 +606,8 @@ fn held_behind_a_gap( let mut sender = session(1); let mut receiver = session(500); handshake(&mut sender, &mut receiver)?; - let _missing = sender.send("gap", b"lost".to_vec(), ordered())?; + let missing = sender.send("gap", b"lost".to_vec(), ordered())?; + sender.receive(&ack_naming(missing.sequence), 1)?; let mut held = 0; for index in 0..frames { let packet = sender.send(&channel_of(index), vec![0], ordered())?; diff --git a/packages/cultnet-rs/tests/rudp_flow_window.rs b/packages/cultnet-rs/tests/rudp_flow_window.rs new file mode 100644 index 00000000..9f10eb50 --- /dev/null +++ b/packages/cultnet-rs/tests/rudp_flow_window.rs @@ -0,0 +1,310 @@ +//! The sender's flow window: a reliable packet goes on the wire only while its +//! sequence is at most 1,023 above the lowest unacked one and the payload above +//! that sequence stays within 4 MiB. Everything else waits in order in the queue. + +use anyhow::Result; +use cultnet_rs::*; + +const CONNECTION_ID: u32 = 0x464c_4f57; +const MIB: usize = 1024 * 1024; + +fn session(initial_sequence: u32) -> CultNetRudpSession { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(initial_sequence), + resend_delay_ms: 25, + max_pending_reliable_packets: None, + }) +} + +/// A sender and a receiver that have shaken hands: the receiver's watermark is +/// seeded by the sender's Connect, and the sender's Connect is acknowledged, so +/// nothing is pending on the sender. +fn connected_pair( + sender_initial_sequence: u32, + receiver_initial_sequence: u32, +) -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut sender = session(sender_initial_sequence); + let mut receiver = session(receiver_initial_sequence); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; + Ok((sender, receiver)) +} + +fn connected(initial_sequence: u32) -> Result { + Ok(connected_pair(initial_sequence, 900)?.0) +} + +fn reliable() -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ..Default::default() + } +} + +/// What `send_many` puts on the wire now; the rest of the packet waits queued. +fn send(session: &mut CultNetRudpSession, payload_bytes: usize) -> Result> { + session.send_many("state", vec![7; payload_bytes], reliable(), None) +} + +fn ack_for(sequence: u32) -> CultNetRudpPacket { + CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Ack, + connection_id: CONNECTION_ID, + sequence: 0, + ack: sequence, + ack_mask: 0, + channel_id: "control".into(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + } +} + +fn sequences(packets: &[CultNetRudpPacket]) -> Vec { + packets.iter().map(|packet| packet.sequence).collect() +} + +/// Sends `g`, then acks every packet up to and including the one 1,023 above +/// it, leaving `g` the only unacked sequence. Returns `g`. +fn lose_g_and_fill_the_span(session: &mut CultNetRudpSession) -> Result { + let g = send(session, 1)?.remove(0); + for offset in 1..=1_023u32 { + let sent = send(session, 1)?; + assert_eq!(sequences(&sent), vec![g.sequence + offset]); + session.receive(&ack_for(g.sequence + offset), 1)?; + } + Ok(g) +} + +#[test] +fn a_lost_packet_holds_the_sender_1023_sequences_ahead_and_is_still_delivered() -> Result<()> { + let (mut sender, mut receiver) = connected_pair(1, 100)?; + let g = send(&mut sender, 1)?.remove(0); + // g is lost. Everything after it is delivered and acked, until the sender stops. + let mut admitted_after_g = 0u32; + for _ in 0..4_200 { + for packet in send(&mut sender, 1)? { + admitted_after_g += 1; + receiver.receive(&packet, 1)?; + sender.receive(&receiver.create_ack(), 1)?; + } + } + assert_eq!(admitted_after_g, 1_023); + assert_eq!(sender.queued_reliable_packet_count(), 4_200 - 1_023); + + let retransmit = sender + .due_resends(1_000) + .into_iter() + .find(|packet| packet.sequence == g.sequence) + .expect("g is still pending and due"); + let delivered = receiver.receive(&retransmit, 1_000)?.delivered; + assert_eq!(delivered.len(), 1); + assert_eq!(delivered[0].sequence, g.sequence); + + let promoted = sender + .receive(&receiver.create_ack_for_received(g.sequence), 1_001)? + .ready_to_send; + assert_eq!(promoted.first().map(|packet| packet.sequence), Some(g.sequence + 1_024)); + Ok(()) +} + +#[test] +fn a_direct_send_past_the_span_is_refused_without_consuming_a_sequence() -> Result<()> { + let mut sender = connected(1)?; + let g = lose_g_and_fill_the_span(&mut sender)?; + assert!(sender.send("state", vec![7], reliable()).is_err()); + assert_eq!(sender.queued_reliable_packet_count(), 0); + sender.receive(&ack_for(g.sequence), 2)?; + let next = sender.send("state", vec![7], reliable())?; + assert_eq!(next.sequence, g.sequence + 1_024); + Ok(()) +} + +#[test] +fn payload_above_the_lowest_unacked_sequence_is_bounded_by_4_mib() -> Result<()> { + let mut sender = connected(1)?; + let g = send(&mut sender, MIB)?.remove(0); + // The lowest unacked packet's own bytes do not count: four more MiB fit above it. + for _ in 0..4 { + assert_eq!(send(&mut sender, MIB)?.len(), 1); + } + assert!(send(&mut sender, MIB)?.is_empty()); + assert!(send(&mut sender, MIB)?.is_empty()); + assert_eq!(sender.queued_reliable_packet_count(), 2); + + // Acking g moves the floor up one packet: exactly one MiB more fits. + let promoted = sender.receive(&ack_for(g.sequence), 1)?.ready_to_send; + assert_eq!(sequences(&promoted), vec![g.sequence + 5]); + assert_eq!(sender.queued_reliable_packet_count(), 1); + Ok(()) +} + +#[test] +fn a_packet_larger_than_the_window_goes_out_alone_and_waits_behind_anything_unacked() -> Result<()> { + let mut sender = connected(1)?; + assert_eq!(send(&mut sender, 5 * MIB)?.len(), 1); + let mut sender = connected(1)?; + let g = send(&mut sender, 1)?.remove(0); + assert!(send(&mut sender, 5 * MIB)?.is_empty()); + let promoted = sender.receive(&ack_for(g.sequence), 1)?.ready_to_send; + assert_eq!(sequences(&promoted), vec![g.sequence + 1]); + Ok(()) +} + +#[test] +fn a_small_packet_does_not_overtake_a_queued_large_one() -> Result<()> { + let mut sender = connected(1)?; + let g = send(&mut sender, 1)?.remove(0); + assert_eq!(send(&mut sender, 3 * MIB)?.len(), 1); + assert!(send(&mut sender, 2 * MIB)?.is_empty()); + // A direct send refuses instead of queueing behind it, and takes no sequence. + assert!(sender.send("state", vec![7], reliable()).is_err()); + assert_eq!(sender.queued_reliable_packet_count(), 1); + // One byte would fit above g, but the 2 MiB packet is ahead of it. + assert!(send(&mut sender, 1)?.is_empty()); + let promoted = sender.receive(&ack_for(g.sequence), 1)?.ready_to_send; + assert_eq!(sequences(&promoted), vec![g.sequence + 2, g.sequence + 3]); + Ok(()) +} + +#[test] +fn acknowledged_bytes_above_a_lost_packet_still_count_against_4_mib() -> Result<()> { + let mut sender = connected(1)?; + let g = send(&mut sender, 1)?.remove(0); + // g is lost. Each 1 MiB frame above it is delivered and acknowledged, and + // the receiver still holds it behind the gap, so it keeps counting. + for offset in 1..=4u32 { + let sent = send(&mut sender, MIB)?; + assert_eq!(sequences(&sent), vec![g.sequence + offset]); + sender.receive(&ack_for(g.sequence + offset), 1)?; + } + assert!(send(&mut sender, MIB)?.is_empty()); + assert_eq!(sender.queued_reliable_packet_count(), 1); + assert!(sender.send("state", vec![7; MIB], reliable()).is_err()); + + // g arrives: nothing is held behind a gap any more, and the queue drains. + let promoted = sender.receive(&ack_for(g.sequence), 2)?.ready_to_send; + assert_eq!(sequences(&promoted), vec![g.sequence + 5]); + // The bytes above the old gap no longer count: another MiB fits. + assert_eq!(sequences(&send(&mut sender, MIB)?), vec![g.sequence + 6]); + Ok(()) +} + +#[test] +fn acknowledged_bytes_count_against_the_lowest_unacked_sequence_as_it_advances() -> Result<()> { + let mut sender = connected(1)?; + let g = send(&mut sender, 1)?.remove(0); + let h = send(&mut sender, 1)?.remove(0); + for _ in 0..3 { + send(&mut sender, MIB)?; + } + // g and the 3 MiB above h are acknowledged; h is lost. + for offset in [2, 3, 4, 0] { + sender.receive(&ack_for(g.sequence + offset), 1)?; + } + assert_eq!(sequences(&send(&mut sender, MIB)?), vec![g.sequence + 5]); + // 4 MiB sit above h, acknowledged or not: one more byte does not fit. + assert!(send(&mut sender, 1)?.is_empty()); + // h arrives, the floor moves to g + 5, and the byte fits. + let promoted = sender.receive(&ack_for(h.sequence), 2)?.ready_to_send; + assert_eq!(sequences(&promoted), vec![g.sequence + 6]); + Ok(()) +} + +#[test] +fn acknowledged_bytes_stop_counting_when_the_lowest_unacked_packet_expires() -> Result<()> { + let mut sender = connected(1)?; + let expiring = CultNetRudpSendOptions { + reliable: true, + now_ms: 0, + reliable_expire_after_ms: Some(100), + ..Default::default() + }; + let g = sender + .send_many("state", vec![7], expiring, None)? + .remove(0); + for offset in 1..=4u32 { + assert_eq!(send(&mut sender, MIB)?.len(), 1); + sender.receive(&ack_for(g.sequence + offset), 1)?; + } + assert!(sender.send("state", vec![7; MIB], reliable()).is_err()); + + // g's deadline passes and it leaves the window; nothing is unacked, so the + // acknowledged bytes no longer sit above any gap. + let at = |now_ms| CultNetRudpSendOptions { + reliable: true, + now_ms, + ..Default::default() + }; + let first = sender.send_many("state", vec![7], at(200), None)?; + assert_eq!(first.len(), 1); + for _ in 0..4 { + assert_eq!(sender.send_many("state", vec![7; MIB], at(200), None)?.len(), 1); + } + Ok(()) +} + +#[test] +fn a_direct_send_is_not_refused_for_bytes_whose_lowest_packet_has_expired() -> Result<()> { + let mut sender = connected(1)?; + let expiring = CultNetRudpSendOptions { + reliable: true, + now_ms: 0, + reliable_expire_after_ms: Some(100), + ..Default::default() + }; + let g = sender + .send_many("state", vec![7], expiring, None)? + .remove(0); + for offset in 1..=4u32 { + assert_eq!(send(&mut sender, MIB)?.len(), 1); + sender.receive(&ack_for(g.sequence + offset), 1)?; + } + assert!(sender.send("state", vec![7; MIB], reliable()).is_err()); + + // The first call after g's deadline is a direct send: it measures the + // window without g, as `send_many` would, and goes on the wire. + let at_200 = CultNetRudpSendOptions { + reliable: true, + now_ms: 200, + ..Default::default() + }; + let sent = sender.send("state", vec![7; MIB], at_200)?; + assert_eq!(sent.sequence, g.sequence + 5); + assert_eq!(sender.outstanding_reliable_packet_count(), 1); + Ok(()) +} + +/// A server that accepts a fresh Connect may send before its Accept is +/// acknowledged, so no acknowledgement of the new generation has moved the +/// floor yet. Bytes the ended generation had acknowledged above its own lost +/// packet belong to no receiver's hold any more and must not count. +#[test] +fn a_new_generation_on_the_server_does_not_inherit_acknowledged_bytes() -> Result<()> { + let mut server = session(500); + let mut first_client = session(1); + let connect = first_client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + server.receive(&ack_for(accept.sequence), 0)?; + let g = send(&mut server, 1)?.remove(0); + for offset in 1..=4u32 { + assert_eq!(sequences(&send(&mut server, MIB)?), vec![g.sequence + offset]); + server.receive(&ack_for(g.sequence + offset), 1)?; + } + assert!(send(&mut server, MIB)?.is_empty()); + + let mut second_client = session(200_000); + let connect = second_client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + // The Accept is unacknowledged and the only thing pending: 1 MiB fits above it. + let sent = server.send("state", vec![7; MIB], reliable())?; + assert_eq!(sent.sequence, accept.sequence + 1); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/schema_discovery_contracts.rs b/packages/cultnet-rs/tests/schema_discovery_contracts.rs index 7c62b80f..833f40c3 100644 --- a/packages/cultnet-rs/tests/schema_discovery_contracts.rs +++ b/packages/cultnet-rs/tests/schema_discovery_contracts.rs @@ -18,85 +18,66 @@ //! Vendoring is a second copy of files this repo already treats as canonical //! (`contracts/cultnet/*.schema.json`, the shared source multiple runtimes compile against per //! `docs/cultnet-selection-cut.md` section 1). A second copy can drift silently, so this test is -//! the thing that makes that impossible without a loud, immediate failure: every vendored file is -//! compared byte-for-byte against its repo-root original. It skips (not fails) when the canonical +//! the thing that makes that impossible without a loud, immediate failure: every `*.schema.json` +//! under the vendored `contracts/cultnet/` is compared byte-for-byte against its repo-root +//! original. The files are found by listing the directory, not from a list kept by hand, so a newly +//! vendored copy cannot escape the check. It skips (not fails) when the canonical //! `contracts/` tree is not present beside the package - i.e. exactly the isolated-copy situation //! R-AB exists to survive - so this check does not reintroduce the same escape it is guarding -//! against. +//! against. A schema embedded by `schema_discovery.rs` but not vendored fails to compile. use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; -/// Every schema this crate's `schema_discovery::builtin_schema_registry` embeds via -/// `include_str!("../contracts/cultnet/…")` - kept in sync with `schema_discovery.rs` by hand; -/// a schema added there and not here is still caught, because the crate would then fail to -/// compile against a file this list never copied to `contracts/cultnet/`. -const VENDORED_SCHEMAS: &[&str] = &[ - "cultnet.hello.schema.json", - "cultnet.document-mutation-contract.schema.json", - "cultnet.transport-profile.schema.json", - "cultnet.login.schema.json", - "cultnet.register.schema.json", - "cultnet.verify.schema.json", - "cultnet.login-success.schema.json", - "cultnet.error.schema.json", - "cultnet.sample-change-name.schema.json", - "cultnet.sample-chat.schema.json", - "cultnet.document-put.schema.json", - "cultnet.document-delete.schema.json", - "cultnet.raw-document-record.schema.json", - "cultnet.document-put-raw.schema.json", - "cultnet.snapshot-request.schema.json", - "cultnet.snapshot-response.schema.json", - "cultnet.snapshot-response-raw.schema.json", - "cultnet.schema-catalog-request.schema.json", - "cultnet.schema-catalog-response.schema.json", - "cultnet.shard-catalog-request.schema.json", - "cultnet.shard-catalog-response.schema.json", - "ghostlight.agent-state.schema.json", -]; - -#[test] -fn vendored_contract_schemas_are_present_and_match_every_include_str_call() { - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let vendored_root = manifest_dir.join("contracts/cultnet"); - for name in VENDORED_SCHEMAS { - let path = vendored_root.join(name); - assert!( - path.is_file(), - "{name} is embedded by schema_discovery.rs but missing from the vendored \ - contracts/cultnet/ copy at {path:?}" - ); +/// Every `*.schema.json` under `root`, as paths relative to it, sorted. +fn schema_files(root: &Path) -> Vec { + fn walk(root: &Path, dir: &Path, found: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|error| panic!("listing {dir:?}: {error}")) { + let path = entry.expect("directory entry").path(); + if path.is_dir() { + walk(root, &path, found); + } else if path.to_string_lossy().ends_with(".schema.json") { + found.push(path.strip_prefix(root).expect("path under root").to_path_buf()); + } + } } + let mut found = Vec::new(); + walk(root, root, &mut found); + found.sort(); + found +} + +fn vendored_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("contracts/cultnet") } #[test] fn vendored_contract_schemas_match_the_canonical_repo_root_source() { - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let vendored_root = manifest_dir.join("contracts/cultnet"); // packages/cultnet-rs -> packages -> repo root -> contracts/cultnet - let canonical_root = manifest_dir.join("../../contracts/cultnet"); + let canonical_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../contracts/cultnet"); if !canonical_root.is_dir() { // R-AB: exactly the isolated-copy case this crate's own vendored files exist to survive // (cargo-mutants, or any tool that copies only this package's directory). Nothing to // compare against, and that is expected, not a failure. eprintln!( - "skipping: {canonical_root:?} not present beside the package (isolated build, \ - the case R-AB's vendored copies are for)" + "skipping: {canonical_root:?} not present beside the package (isolated build, the case R-AB's vendored copies are for)" ); return; } - for name in VENDORED_SCHEMAS { - let canonical = fs::read_to_string(canonical_root.join(name)) - .unwrap_or_else(|error| panic!("reading canonical contracts/cultnet/{name}: {error}")); - let vendored = fs::read_to_string(vendored_root.join(name)) - .unwrap_or_else(|error| panic!("reading vendored contracts/cultnet/{name}: {error}")); + let vendored = schema_files(&vendored_root()); + assert!(!vendored.is_empty(), "no vendored schemas found under {:?}", vendored_root()); + for name in vendored { + let shown = name.display(); + let canonical = fs::read_to_string(canonical_root.join(&name)).unwrap_or_else(|error| { + panic!("packages/cultnet-rs/contracts/cultnet/{shown} has no canonical contracts/cultnet/{shown}: {error}") + }); + let copy = fs::read_to_string(vendored_root().join(&name)) + .unwrap_or_else(|error| panic!("reading vendored contracts/cultnet/{shown}: {error}")); assert_eq!( - canonical, vendored, - "packages/cultnet-rs/contracts/cultnet/{name} has drifted from the canonical \ - contracts/cultnet/{name} - re-copy it" + canonical, copy, + "packages/cultnet-rs/contracts/cultnet/{shown} has drifted from the canonical contracts/cultnet/{shown} - re-copy it" ); } } @@ -107,7 +88,7 @@ fn builtin_schema_registry_loads_every_vendored_schema() { let listed = registry.list(&cultnet_rs::CultNetSchemaCatalogOptions::default()); assert_eq!( listed.len(), - VENDORED_SCHEMAS.len(), + schema_files(&vendored_root()).len(), "one registration per vendored schema file" ); } diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 8950f638..1e856822 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -106,6 +106,9 @@ export { createRudpTransportProfile, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, + rudpClientBindHost, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetRudpReconnectLoop, CultNetRudpSession, diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 606cd418..90760591 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -12,6 +12,9 @@ import { CultNetRudpSocketTransportConnection, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, + rudpClientBindHost, type CultNetRudpPacket, } from "./rudp"; import { CultNetPeer } from "./peer"; @@ -38,6 +41,12 @@ export interface CultNetOperationServer { readonly endpoint: string; /** Datagrams read and discarded: malformed, unadmitted, refused by their session, or failed in handling. */ readonly packetsDropped: number; + /** + * Datagrams that could not be sent to a peer and may yet be (no route, full buffers): + * each is that peer's lost datagram, resent if it was reliable. One that can never be + * sent as built ends that peer's session instead and is not counted here. + */ + readonly sendFailures: number; close(): Promise; } @@ -55,8 +64,12 @@ interface RemoteSession { handling: number; /** When the last handler finished: the response still needs its acknowledgement. */ lastHandledAtMs: number; + /** A packet it owed could never be sent, so the session ended. */ + unsendable: boolean; } +type SendPacket = (key: string, peer: RemoteSession, packet: CultNetRudpPacket) => void; + export async function startCultNetOperationServer( options: CultNetOperationServerOptions, ): Promise { @@ -65,16 +78,32 @@ export async function startCultNetOperationServer( const socket = createSocket("udp4"); const connectionId = options.connectionId ?? DEFAULT_CONNECTION_ID; const sessions = new Map(); - const sendPacket = (remote: RemoteInfo, packet: CultNetRudpPacket): void => { - const wire = encodeRudpPacket(packet); - socket.send(wire, remote.port, remote.address); - }; let packetsDropped = 0; + let sendFailures = 0; + const sendPacket: SendPacket = (key, peer, packet) => { + const generation = peer.session.generation; + sendRudpDatagram(socket, encodeRudpPacket(packet), peer.remote.port, peer.remote.address, (error) => { + if (!isPermanentSendError(error)) { + sendFailures += 1; + return; + } + // A failure reported after the generation that sent the datagram ended is owed by + // nothing: that generation's end already told the peer, and a client that connected + // again from the endpoint owns it now. A live generation is the one in `sessions`. + if (peer.session.generation !== generation) return; + // The datagram can never be sent as built: the session that owes it ends, never + // the server, and the peer is told. + peer.unsendable = true; + sessions.delete(key); + const goodbye = encodeRudpPacket(peer.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); + }); + }; socket.on("message", (wire, remote) => { // What a datagram carries is the sender's business: a rejection here must // drop and count the packet, never become an unhandled rejection that ends // the process. - handleServerDatagram(sessions, connectionId, options, wire, remote, sendPacket).then( + handleServerDatagram(sessions, connectionId, options, wire, remote, socket, sendPacket).then( admitted => { if (!admitted) packetsDropped += 1; }, () => { packetsDropped += 1; }, ); @@ -89,8 +118,8 @@ export async function startCultNetOperationServer( if (peer.handling > 0 || now - peer.lastHandledAtMs < idleTimeoutMs) continue; if (peer.session.checkTimeout(now, idleTimeoutMs)) sessions.delete(key); } - for (const peer of sessions.values()) { - for (const packet of peer.session.dueResends(Date.now())) sendPacket(peer.remote, packet); + for (const [key, peer] of sessions) { + for (const packet of peer.session.dueResends(Date.now())) sendPacket(key, peer, packet); } }, 25); resendTimer.unref?.(); @@ -99,6 +128,7 @@ export async function startCultNetOperationServer( return { endpoint, get packetsDropped() { return packetsDropped; }, + get sendFailures() { return sendFailures; }, close: async () => { clearInterval(resendTimer); await closeSocket(socket); @@ -112,8 +142,9 @@ export async function invokeCultNetOperation( options: CultNetOperationClientOptions, ): Promise { const target = parseRudpEndpoint(endpoint); - const socket = createSocket("udp4"); - await bindSocket(socket, 0, "127.0.0.1"); + const bindHost = await rudpClientBindHost(target.host); + const socket = createSocket(bindHost.includes(":") ? "udp6" : "udp4"); + await bindSocket(socket, 0, bindHost); const transport = new CultNetRudpSocketTransportConnection({ mode: "client", runtimeId: options.runtimeId, @@ -153,7 +184,8 @@ async function handleServerDatagram( options: CultNetOperationServerOptions, wire: Buffer, remote: RemoteInfo, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + socket: Socket, + sendPacket: SendPacket, ): Promise { let packet: CultNetRudpPacket; try { @@ -165,14 +197,24 @@ async function handleServerDatagram( const key = `${remote.address}:${remote.port}`; let peer = sessions.get(key); if (packet.packetType === "connect") { - if (!peer) { - if (sessions.size >= MAX_OPERATION_SESSIONS) return false; - peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote, handling: 0, lastHandledAtMs: 0 }; - sessions.set(key, peer); + if (peer) { + // A Connect from an admitted peer repeats: the session answers with the + // Accept already owed and queues nothing. + sendPacket(key, peer, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + return true; } - // A Connect from an admitted peer repeats: the session answers with the - // Accept already owed and queues nothing. - sendPacket(remote, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + if (sessions.size >= MAX_OPERATION_SESSIONS) return false; + const candidate: RemoteSession = { + session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), + remote, + handling: 0, + lastHandledAtMs: 0, + unsendable: false, + }; + sendPacket(key, candidate, candidate.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + // A peer whose Accept can never be sent cannot be answered, so no session starts. + if (candidate.unsendable) return false; + sessions.set(key, candidate); return true; } if (!peer) return false; @@ -180,15 +222,20 @@ async function handleServerDatagram( try { result = peer.session.receive(packet, Date.now()); } catch { - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } - if (result.reply) sendPacket(remote, result.reply); - for (const ready of result.readyToSend ?? []) sendPacket(remote, ready); + if (result.reply) sendPacket(key, peer, result.reply); + for (const ready of result.readyToSend ?? []) sendPacket(key, peer, ready); if (result.disconnected) { sessions.delete(key); return true; } + // A request belongs to the session generation it arrived on. If that generation ends while a + // handler runs (a goodbye, a refusal, a timeout, a new Connect from the same endpoint), what + // the handler returns or throws belongs to no live session, and nothing of it may reach the + // endpoint: a client that connected again there owns it now. + const generation = peer.session.generation; for (const frame of result.delivered) { if (frame.channelId !== "schema") continue; try { @@ -202,22 +249,24 @@ async function handleServerDatagram( peer.handling -= 1; peer.lastHandledAtMs = Date.now(); } + if (peer.session.generation !== generation) return true; const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); for (const responsePacket of peer.session.sendMany("schema", payload, { reliable: true, ordered: true, nowMs: Date.now(), maxFragmentBytes: options.maxFragmentBytes ?? 2048, - })) sendPacket(remote, responsePacket); + })) sendPacket(key, peer, responsePacket); } catch { + if (peer.session.generation !== generation) return true; // The session recorded this request's sequence, so keeping it would // acknowledge the retransmit of a request that was never handled. - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } } if (packet.packetType === "data" || result.delivered.length > 0) { - sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); + sendPacket(key, peer, peer.session.createAckForReceived(packet.sequence)); } return true; } @@ -225,17 +274,19 @@ async function handleServerDatagram( /** * Ends a session that took a packet it could not serve, and tells the peer. The * reset comes first, or the goodbye's ack field would acknowledge the very - * packet that was refused. A session already replaced under the same key is left. + * packet that was refused. The goodbye is best-effort: the session ends whether + * or not the peer hears it. */ function endSession( sessions: Map, key: string, peer: RemoteSession, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + socket: Socket, ): void { - if (sessions.get(key) === peer) sessions.delete(key); + sessions.delete(key); peer.session.resetPeerState(); - sendPacket(peer.remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + const goodbye = encodeRudpPacket(peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index e32bb2d7..e68293d3 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -1,8 +1,22 @@ import { randomInt } from "node:crypto"; import { EventEmitter } from "node:events"; import { type RemoteInfo, type Socket } from "node:dgram"; +import { lookup } from "node:dns/promises"; import type { CultNetTransportProfile } from "./contracts"; + +/** + * The address an RUDP client binds when its caller names none: loopback for a + * loopback endpoint, otherwise the unspecified address of the endpoint's family. + * A socket bound to loopback cannot send off the host on Windows. + */ +export async function rudpClientBindHost(endpointHost: string): Promise { + const { address, family } = await lookup(endpointHost.replace(/^\[(.*)\]$/, "$1")); + if (family === 6) { + return address === "::1" ? "::1" : "::"; + } + return address.startsWith("127.") ? "127.0.0.1" : "0.0.0.0"; +} import { CultNetReconnectController, createCultNetReconnectPolicy, @@ -16,9 +30,18 @@ import { const RUDP_MAGIC = [0x43, 0x4e, 0x52, 0x30] as const; // CNR0 const RUDP_VERSION = 0; const RUDP_FIXED_HEADER_BYTES = 36; +/// The largest UDP payload a datagram can carry: 65,535 less the UDP header and, +/// over IPv4, the minimal IP header. IPv6's header is outside its payload length. +const MAX_UDP4_DATAGRAM_BYTES = 65_507; +const MAX_UDP6_DATAGRAM_BYTES = 65_527; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; +// Flow window: a reliable packet is admitted only while its sequence is at most RUDP_FLOW_WINDOW_SEQUENCES above the +// lowest unacked one and the payload above that sequence stays within RUDP_FLOW_WINDOW_BYTES. +const RUDP_FLOW_WINDOW_SEQUENCES = 1_023; +const RUDP_FLOW_WINDOW_BYTES = 4 * 1024 * 1024; + /// How long a client keeps retransmitting a Connect nobody answered before it /// abandons that attempt for a fresh one. A Connect the server answers with an /// Ack, never an Accept, is one the server judged stale (see `acceptConnect`); @@ -198,6 +221,9 @@ export class CultNetRudpSession { readonly #receivedSequences = new Set(); readonly #latestSequencedByChannel = new Map(); readonly #pendingReliable = new Map(); + /// Payload sizes of acknowledged reliable sequences above the lowest unacknowledged one. The receiver still holds + /// those bytes behind the gap, so the flow window keeps counting them until the lowest sequence passes them. + readonly #ackedAboveLowest = new Map(); readonly #queuedReliable: CultNetRudpPacket[] = []; /// Every reliable sequence up to and including this one has been received /// since the peer state was last reset. Only the handshake seeds it: the @@ -280,6 +306,7 @@ export class CultNetRudpSession { this.#awaitingAccept = false; this.#generation += 1; this.#pendingReliable.clear(); + this.#ackedAboveLowest.clear(); this.#queuedReliable.splice(0); } @@ -406,7 +433,7 @@ export class CultNetRudpSession { payload: Uint8Array, options: { reliable?: boolean; ordered?: boolean; sequenced?: boolean; nowMs?: number } = {}, ): CultNetRudpPacket { - if (options.reliable && this.#pendingReliable.size >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS) { + if (options.reliable && (this.#queuedReliable.length > 0 || !this.#windowAdmits(this.#nextSequence, payload.length))) { throw new Error("RUDP reliable send window is full; receive acknowledgements before sending."); } return this.sendMany(channelId, payload, options)[0]!; @@ -645,6 +672,18 @@ export class CultNetRudpSession { }); } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `isPermanentSendError`). Resending it would fail forever, and + /// dropping it would leave a reliable sequence the peer waits on for good, so + /// the session cannot be kept. Returns the goodbye for the peer; its reason + /// names the error's code, never its message, which can quote the address. + endUnsendableSession(error: Error): CultNetRudpPacket { + this.resetPeerState(); + return this.createDisconnect( + Buffer.from(`${RUDP_UNSENDABLE_PACKET_REASON}: ${sendErrorCode(error)}`, "utf8"), + ); + } + checkTimeout(nowMs: number, timeoutMs: number): boolean { if (!this.#connected || this.#lastReceivedAtMs === undefined) { return false; @@ -739,18 +778,44 @@ export class CultNetRudpSession { }); } + /** + * Whether a reliable packet may go on the wire now: the window has a slot, and its sequence and the payload above + * the lowest unacked sequence stay inside the flow window. With nothing pending, any packet is admissible. + */ + #windowAdmits(sequence: number, payloadLength: number): boolean { + if (this.#pendingReliable.size >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS) return false; + if (this.#pendingReliable.size === 0) return true; + const lowest = Math.min(...this.#pendingReliable.keys()); + let bytesAbove = 0; + for (const [pendingSequence, pending] of this.#pendingReliable) { + if (pendingSequence > lowest) bytesAbove += pending.packet.payload?.length ?? 0; + } + for (const length of this.#ackedAboveLowest.values()) bytesAbove += length; + return sequence - lowest <= RUDP_FLOW_WINDOW_SEQUENCES && bytesAbove + payloadLength <= RUDP_FLOW_WINDOW_BYTES; + } + #admitReliablePackets(packets: CultNetRudpPacket[], nowMs: number): CultNetRudpPacket[] { - const available = Math.max(0, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS - this.#pendingReliable.size); - const ready = packets.slice(0, available); - for (const packet of ready) this.#trackReliable(packet, nowMs); - this.#queuedReliable.push(...packets.slice(available)); + const ready: CultNetRudpPacket[] = []; + for (const packet of packets) { + if (this.#queuedReliable.length === 0 && this.#windowAdmits(packet.sequence, packet.payload?.length ?? 0)) { + this.#trackReliable(packet, nowMs); + ready.push(packet); + } else { + this.#queuedReliable.push(packet); + } + } return ready; } #promoteQueuedReliable(nowMs: number): CultNetRudpPacket[] { - const available = Math.max(0, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS - this.#pendingReliable.size); - const ready = this.#queuedReliable.splice(0, available); - for (const packet of ready) this.#trackReliable(packet, nowMs); + const ready: CultNetRudpPacket[] = []; + while (this.#queuedReliable.length > 0) { + const packet = this.#queuedReliable[0]; + if (!this.#windowAdmits(packet.sequence, packet.payload?.length ?? 0)) break; + this.#queuedReliable.shift(); + this.#trackReliable(packet, nowMs); + ready.push(packet); + } return ready; } @@ -764,12 +829,33 @@ export class CultNetRudpSession { } #applyAcknowledgements(packet: CultNetRudpPacket): void { - this.#pendingReliable.delete(packet.ack); + this.#acknowledge(packet.ack); for (let bit = 0; bit < 32; bit += 1) { if ((packet.ackMask & (1 << bit)) !== 0) { - this.#pendingReliable.delete(packet.ack - bit - 1); + this.#acknowledge(packet.ack - bit - 1); } } + this.#forgetAckedBelowLowest(); + } + + #acknowledge(sequence: number): void { + const pending = this.#pendingReliable.get(sequence); + if (pending === undefined) return; + this.#pendingReliable.delete(sequence); + this.#ackedAboveLowest.set(sequence, pending.packet.payload?.length ?? 0); + } + + /// Acknowledged sizes stop counting once the lowest unacknowledged sequence passes them, and all of them stop + /// when nothing is unacknowledged. + #forgetAckedBelowLowest(): void { + if (this.#pendingReliable.size === 0) { + this.#ackedAboveLowest.clear(); + return; + } + const lowest = Math.min(...this.#pendingReliable.keys()); + for (const sequence of this.#ackedAboveLowest.keys()) { + if (sequence <= lowest) this.#ackedAboveLowest.delete(sequence); + } } /// The handshake's one act on the watermark: the peer's Connect or Accept is @@ -955,6 +1041,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement readonly #mode: "client" | "server"; readonly #resendTimer: NodeJS.Timeout; readonly #maxFragmentBytes: number | undefined; + readonly #maxDatagramBytes: number; #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; @@ -967,6 +1054,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(options: CultNetRudpSocketTransportOptions) { @@ -985,6 +1073,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement const address = this.#socket.address(); const localPort = typeof address === "string" ? undefined : address.port; const localHost = typeof address === "string" ? undefined : address.address; + this.#maxDatagramBytes = + typeof address !== "string" && String(address.family).endsWith("6") ? MAX_UDP6_DATAGRAM_BYTES : MAX_UDP4_DATAGRAM_BYTES; this.profile = createRudpTransportProfile(options.runtimeId, { transportId: options.transportId, host: localHost, @@ -1026,6 +1116,15 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } send(channelId: string, payload: Uint8Array): void { + // A datagram larger than UDP can carry can never be sent, and Node reports that + // only later, to the send's callback. Checked here it is the caller's error, as + // in the other runtimes: the send throws, and nothing of it is queued or takes a + // sequence. Every datagram of a send is at most one fragment of its payload. + const largestPayload = + this.#maxFragmentBytes === undefined ? payload.byteLength : Math.min(payload.byteLength, this.#maxFragmentBytes); + if (RUDP_FIXED_HEADER_BYTES + Buffer.byteLength(channelId, "utf8") + largestPayload > this.#maxDatagramBytes) { + throw Object.assign(new Error("RUDP datagram is larger than UDP can carry."), { code: "EMSGSIZE" }); + } const packets = this.#session.sendMany(channelId, payload, { ...channelOptions(channelId), nowMs: Date.now(), @@ -1100,11 +1199,22 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); + this.#endWithGoodbye(this.#session.createDisconnect(reason)); + } + + /** The session owes its peer a packet that can never be sent as built. */ + #endUnsendableSession(error: Error): void { + this.#endWithGoodbye(this.#session.endUnsendableSession(error)); + } + + #endWithGoodbye(goodbye: CultNetRudpPacket): void { + const reason = goodbye.payload; this.#endedReason = reason; - try { - this.#sendPacket(this.#session.createDisconnect(reason)); - } catch { + if (this.#remoteHost && this.#remotePort !== undefined) { // Best-effort: the session ends whether or not the peer hears it. + const wire = encodeRudpPacket(goodbye); + this.#stats.bytesSent += wire.length; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, () => {}); } if (this.#mode === "server") { // Only a Connect can claim the endpoint again. @@ -1175,7 +1285,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#stats.packetsDropped += 1; return; } - this.#sendPacket(accept); + this.#sendInPoll(accept); return; } @@ -1192,10 +1302,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } try { if (result.reply) { - this.#sendPacket(result.reply); + this.#sendInPoll(result.reply); } for (const ready of result.readyToSend ?? []) { - this.#sendPacket(ready); + this.#sendInPoll(ready); } if (result.pong) { this.emit("pong", { payload: result.pongPayload ?? new Uint8Array() }); @@ -1214,7 +1324,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } if (packet.packetType === "accept" || packet.packetType === "data" || result.delivered.length > 0) { - this.#sendPacket(this.#session.createAckForReceived(packet.sequence)); + this.#sendInPoll(this.#session.createAckForReceived(packet.sequence)); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); @@ -1222,18 +1332,63 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } #sendDueResends(): void { + const generation = this.#session.generation; for (const packet of this.#session.dueResends(Date.now())) { - this.#sendPacket(packet); + this.#sendInPoll(packet); + // A resend that can never be sent ended the session and what it owed. + if (this.#session.generation !== generation) return; } } + /** A caller-directed send: a failure the send throws is the caller's error. */ #sendPacket(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) throw thrown; + } + + /** + * A send with no caller to hand a failure to (a receive or a resend): the failure + * is handled as every later one is, and never escapes the timer or the socket's + * "message" handler. + */ + #sendInPoll(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) this.#sendFailed(thrown); + } + + /** Sends one packet and returns the failure the send threw, if any. */ + #transmit(packet: CultNetRudpPacket): Error | undefined { if (!this.#remoteHost || this.#remotePort === undefined) { throw new Error("RUDP socket transport does not have a remote endpoint."); } const wire = encodeRudpPacket(packet); this.#stats.bytesSent += wire.length; - this.#socket.send(wire, this.#remotePort, this.#remoteHost); + let thrown: Error | undefined; + let sending = true; + const generation = this.#session.generation; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, (error) => { + if (sending) { + thrown = error; + } else if (this.#session.generation === generation) { + // A later failure belongs to the session generation that sent it. + this.#sendFailed(error); + } + }); + sending = false; + return thrown; + } + + /** + * A datagram that can never be sent ends the session; any other failure is a lost + * datagram, counted and never emitted. Node reports a failed send only to its callback, + * not on the socket's "error" event, and an "error" with no listener ends the process. + */ + #sendFailed(error: Error): void { + if (isPermanentSendError(error)) { + this.#endUnsendableSession(error); + return; + } + this.#stats.sendFailures += 1; } } @@ -1327,6 +1482,61 @@ export class CultNetRudpReconnectLoop extends EventEmitter { } } +/** Prefix of the goodbye reason for a session that owed a packet which can never be sent. */ +const RUDP_UNSENDABLE_PACKET_REASON = "packet could not be sent"; + +/** + * Failures that can never pass for the datagram as built: too large for a datagram + * (`EMSGSIZE`), an address malformed for the socket (`EINVAL`, and Node's own + * `ERR_SOCKET_BAD_PORT`, which it throws for port 0 before the system would say + * `EINVAL`), or an address of another family (`EAFNOSUPPORT`). libuv names these the + * same on every platform. + */ +const PERMANENT_SEND_ERROR_CODES: ReadonlySet = new Set([ + "EMSGSIZE", + "EINVAL", + "EAFNOSUPPORT", + "ERR_SOCKET_BAD_PORT", +]); + +function sendErrorCode(error: Error): string { + const code = (error as NodeJS.ErrnoException).code; + return typeof code === "string" && code.length > 0 ? code : "UNKNOWN"; +} + +/** + * Whether a failed UDP send can never succeed for the datagram as built, so resending it is + * pointless and the session that owes it cannot be kept. Everything else (no route, full + * buffers, a closed socket, an interrupted call) is a property of the path or the moment: a + * lost datagram, not a reason to end a session. + */ +export function isPermanentSendError(error: Error): boolean { + return PERMANENT_SEND_ERROR_CODES.has(sendErrorCode(error)); +} + +/** + * Sends one datagram. Node reports a failed send two ways: it throws for an argument it + * refuses (port 0 among them) and passes a system failure to the callback later. Both + * reach `onFailure` and nothing else, so a datagram that cannot be sent to one peer never + * escapes a timer or a receive handler, and never reaches the socket's "error" event. A + * failure the send throws reaches `onFailure` before this returns. + */ +export function sendRudpDatagram( + socket: Socket, + wire: Uint8Array, + port: number, + address: string, + onFailure: (error: Error) => void, +): void { + try { + socket.send(wire, port, address, (error) => { + if (error) onFailure(error); + }); + } catch (error) { + onFailure(error instanceof Error ? error : new Error(String(error))); + } +} + export function createRudpTransportProfile( runtimeId: string, options: RudpTransportProfileOptions = {}, diff --git a/packages/cultnet-ts/src/transport.ts b/packages/cultnet-ts/src/transport.ts index f900aa23..02f5e8c1 100644 --- a/packages/cultnet-ts/src/transport.ts +++ b/packages/cultnet-ts/src/transport.ts @@ -16,6 +16,13 @@ export interface CultNetTransportStats { framesSent: number; /** Datagrams read and discarded because they belong to no session on this transport. */ packetsDropped: number; + /** + * Datagrams this transport could not send and treats as lost (no route, full buffers): + * a reliable packet stays pending and is resent. A send a caller made throws its failure + * to the caller instead, and a datagram that can never be sent as built ends the session. + * Zero for stream transports. + */ + sendFailures: number; } export interface CultNetReconnectPolicy { @@ -195,6 +202,7 @@ export class TcpFramedTransportConnection extends EventEmitter implements CultNe framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(stream: Duplex, profile: CultNetTransportProfile) { diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 4a0c3bbc..a6f19800 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -5,7 +5,7 @@ import { Duplex } from "node:stream"; import dgram, { type Socket } from "node:dgram"; import { rmSync, mkdtempSync } from "node:fs"; import { join } from "node:path"; -import { tmpdir } from "node:os"; +import { networkInterfaces, tmpdir } from "node:os"; import { z } from "zod"; import { decode as decodeMsgpack, encode } from "@msgpack/msgpack"; @@ -47,6 +47,7 @@ import { ghostlightAgentStateGeneratedContract, parseCultNetMessage, invokeCultNetOperation, + rudpClientBindHost, startCultNetOperationServer, validateGhostlightAgentStateGenerated, validateGhostlightAgentState, @@ -1416,6 +1417,167 @@ test("rudp session advances large fragment sets through a bounded reliable windo assert.equal(oldAck.ackMask, 0); }); +// The sender's flow window: a reliable packet goes on the wire only while its sequence is at most 1,023 above the +// lowest unacked one and the payload above that sequence stays within 4 MiB. +const FLOW_CONNECTION_ID = 0x464c4f57; +const MIB = 1024 * 1024; + +function flowSession(initialSequence: number): CultNetRudpSession { + return new CultNetRudpSession({ connectionId: FLOW_CONNECTION_ID, initialSequence, resendDelayMs: 25 }); +} + +// A sender and a receiver that have shaken hands: the receiver's watermark is seeded by the sender's Connect, and the +// sender's Connect is acknowledged, so nothing is pending on the sender. +function connectedFlowPair(senderInitialSequence: number, receiverInitialSequence: number): [CultNetRudpSession, CultNetRudpSession] { + const sender = flowSession(senderInitialSequence); + const receiver = flowSession(receiverInitialSequence); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); + return [sender, receiver]; +} + +function connectedFlowSession(initialSequence: number): CultNetRudpSession { + return connectedFlowPair(initialSequence, 900)[0]; +} + +function flowSend(session: CultNetRudpSession, payloadBytes: number): CultNetRudpPacket[] { + return session.sendMany("state", new Uint8Array(payloadBytes), { reliable: true }); +} + +function flowAck(sequence: number): CultNetRudpPacket { + return { packetType: "ack", connectionId: FLOW_CONNECTION_ID, sequence: 0, ack: sequence, ackMask: 0, channelId: "control" }; +} + +const sequencesOf = (packets: CultNetRudpPacket[]): number[] => packets.map((packet) => packet.sequence); + +function loseGAndFillTheSpan(session: CultNetRudpSession): CultNetRudpPacket { + const g = flowSend(session, 1)[0]!; + for (let offset = 1; offset <= 1023; offset++) { + assert.deepEqual(sequencesOf(flowSend(session, 1)), [g.sequence + offset]); + session.receive(flowAck(g.sequence + offset), 1); + } + return g; +} + +test("a lost packet holds the sender 1023 sequences ahead and is still delivered", () => { + const [sender, receiver] = connectedFlowPair(1, 100); + const g = flowSend(sender, 1)[0]!; + let admittedAfterG = 0; + for (let index = 0; index < 4200; index++) { + for (const packet of flowSend(sender, 1)) { + admittedAfterG++; + receiver.receive(packet, 1); + sender.receive(receiver.createAck(), 1); + } + } + assert.equal(admittedAfterG, 1023); + assert.equal(sender.queuedReliablePacketCount, 4200 - 1023); + + const retransmit = sender.dueResends(1000).find((packet) => packet.sequence === g.sequence)!; + const delivered = receiver.receive(retransmit, 1000).delivered; + assert.deepEqual(delivered.map((frame) => frame.sequence), [g.sequence]); + + const promoted = sender.receive(receiver.createAckForReceived(g.sequence), 1001).readyToSend ?? []; + assert.equal(promoted[0]!.sequence, g.sequence + 1024); +}); + +test("a direct send past the span is refused without consuming a sequence", () => { + const sender = connectedFlowSession(1); + const g = loseGAndFillTheSpan(sender); + assert.throws(() => sender.send("state", new Uint8Array(1), { reliable: true })); + assert.equal(sender.queuedReliablePacketCount, 0); + sender.receive(flowAck(g.sequence), 2); + assert.equal(sender.send("state", new Uint8Array(1), { reliable: true }).sequence, g.sequence + 1024); +}); + +test("payload above the lowest unacked sequence is bounded by 4 MiB", () => { + const sender = connectedFlowSession(1); + const g = flowSend(sender, MIB)[0]!; + // The lowest unacked packet's own bytes do not count: four more MiB fit above it. + for (let index = 0; index < 4; index++) assert.equal(flowSend(sender, MIB).length, 1); + assert.equal(flowSend(sender, MIB).length, 0); + assert.equal(flowSend(sender, MIB).length, 0); + assert.equal(sender.queuedReliablePacketCount, 2); + + // Acking g moves the floor up one packet: exactly one MiB more fits. + const promoted = sender.receive(flowAck(g.sequence), 1).readyToSend ?? []; + assert.deepEqual(sequencesOf(promoted), [g.sequence + 5]); + assert.equal(sender.queuedReliablePacketCount, 1); +}); + +test("a packet larger than the window goes out alone and waits behind anything unacked", () => { + assert.equal(flowSend(connectedFlowSession(1), 5 * MIB).length, 1); + const sender = connectedFlowSession(1); + const g = flowSend(sender, 1)[0]!; + assert.equal(flowSend(sender, 5 * MIB).length, 0); + assert.deepEqual(sequencesOf(sender.receive(flowAck(g.sequence), 1).readyToSend ?? []), [g.sequence + 1]); +}); + +test("a small packet does not overtake a queued large one", () => { + const sender = connectedFlowSession(1); + const g = flowSend(sender, 1)[0]!; + assert.equal(flowSend(sender, 3 * MIB).length, 1); + assert.equal(flowSend(sender, 2 * MIB).length, 0); + // A direct send refuses instead of queueing behind it, and takes no sequence. + assert.throws(() => sender.send("state", new Uint8Array(1), { reliable: true })); + assert.equal(sender.queuedReliablePacketCount, 1); + // One byte would fit above g, but the 2 MiB packet is ahead of it. + assert.equal(flowSend(sender, 1).length, 0); + assert.deepEqual(sequencesOf(sender.receive(flowAck(g.sequence), 1).readyToSend ?? []), [g.sequence + 2, g.sequence + 3]); +}); + +test("acknowledged bytes above a lost packet still count against 4 MiB", () => { + const sender = connectedFlowSession(1); + const g = flowSend(sender, 1)[0]!; + // g is lost. Each 1 MiB frame above it is delivered and acknowledged, and the receiver still holds it behind the + // gap, so it keeps counting. + for (let offset = 1; offset <= 4; offset++) { + assert.deepEqual(sequencesOf(flowSend(sender, MIB)), [g.sequence + offset]); + sender.receive(flowAck(g.sequence + offset), 1); + } + assert.equal(flowSend(sender, MIB).length, 0); + assert.equal(sender.queuedReliablePacketCount, 1); + assert.throws(() => sender.send("state", new Uint8Array(MIB), { reliable: true })); + + // g arrives: nothing is held behind a gap any more, and the queue drains. + assert.deepEqual(sequencesOf(sender.receive(flowAck(g.sequence), 2).readyToSend ?? []), [g.sequence + 5]); + // The bytes above the old gap no longer count: another MiB fits. + assert.deepEqual(sequencesOf(flowSend(sender, MIB)), [g.sequence + 6]); +}); + +test("acknowledged bytes count against the lowest unacked sequence as it advances", () => { + const sender = connectedFlowSession(1); + const g = flowSend(sender, 1)[0]!; + const h = flowSend(sender, 1)[0]!; + for (let index = 0; index < 3; index++) flowSend(sender, MIB); + // g and the 3 MiB above h are acknowledged; h is lost. + for (const offset of [2, 3, 4, 0]) sender.receive(flowAck(g.sequence + offset), 1); + assert.deepEqual(sequencesOf(flowSend(sender, MIB)), [g.sequence + 5]); + // 4 MiB sit above h, acknowledged or not: one more byte does not fit. + assert.equal(flowSend(sender, 1).length, 0); + // h arrives, the floor moves to g + 5, and the byte fits. + assert.deepEqual(sequencesOf(sender.receive(flowAck(h.sequence), 2).readyToSend ?? []), [g.sequence + 6]); +}); + +// A server that accepts a fresh Connect may send before its Accept is acknowledged, so no acknowledgement of the new +// generation has moved the floor yet. Bytes the ended generation had acknowledged above its own lost packet belong to +// no receiver's hold any more and must not count. +test("a new generation on the server does not inherit acknowledged bytes", () => { + const server = flowSession(500); + const accept = server.acceptConnect(flowSession(1).createConnect(0), 0); + server.receive(flowAck(accept.sequence), 0); + const g = flowSend(server, 1)[0]!; + for (let offset = 1; offset <= 4; offset++) { + assert.deepEqual(sequencesOf(flowSend(server, MIB)), [g.sequence + offset]); + server.receive(flowAck(g.sequence + offset), 1); + } + assert.equal(flowSend(server, MIB).length, 0); + + const nextAccept = server.acceptConnect(flowSession(200_000).createConnect(2), 2); + assert.equal(nextAccept.packetType, "accept"); + // The Accept is unacknowledged and the only thing pending: 1 MiB fits above it. + assert.equal(server.send("state", new Uint8Array(MIB), { reliable: true }).sequence, nextAccept.sequence + 1); +}); + test("CultNet contracts encode legacy bytes without Node Buffer authority", () => { const originalBuffer = globalThis.Buffer; try { @@ -2224,6 +2386,47 @@ test("operation service frees the slots of abandoned sessions once they idle out } }); +test("an RUDP client binds loopback only for a loopback endpoint", async () => { + assert.equal(await rudpClientBindHost("10.77.0.1"), "0.0.0.0"); + assert.equal(await rudpClientBindHost("2001:db8::1"), "::"); + assert.equal(await rudpClientBindHost("[2001:db8::1]"), "::"); + assert.equal(await rudpClientBindHost("127.0.0.1"), "127.0.0.1"); + assert.equal(await rudpClientBindHost("127.0.0.2"), "127.0.0.1"); + assert.equal(await rudpClientBindHost("::1"), "::1"); + assert.equal(await rudpClientBindHost("[::1]"), "::1"); +}); + +// A socket bound to loopback cannot send off the host on Windows. On Linux it +// reaches a local non-loopback address, but from 127.0.0.1, so the source +// address shows which one the client bound. +test("an operation client for a non-loopback endpoint does not send from loopback", async (t) => { + const address = Object.values(networkInterfaces()) + .flat() + .find((entry) => entry && entry.family === "IPv4" && !entry.internal)?.address; + if (!address) { + t.skip("no non-loopback IPv4 interface on this host"); + return; + } + const listener = dgram.createSocket("udp4"); + await new Promise((resolve) => listener.bind(0, "0.0.0.0", resolve)); + const source = new Promise((resolve) => listener.once("message", (_wire, remote) => resolve(remote.address))); + const request: CultNetOperationRequestMessage = { + schemaVersion: "cultnet.operation_request.v0", + messageId: "bind-host", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }; + try { + await assert.rejects(invokeCultNetOperation(`rudp://${address}:${udpPort(listener)}`, request, { runtimeId: "bind-test", timeoutMs: 300 })); + assert.equal(await source, address); + } finally { + listener.close(); + } +}); + test("server-mode transport replaces its peer when a Connect arrives from another endpoint", async () => { const serverSocket = await bindUdpSocket(); const socketA = await bindUdpSocket(); @@ -2622,6 +2825,89 @@ test("server-mode transport admits a restarted client and keeps its session when } }); +// A request belongs to the session generation it arrived on. A client that connects again +// from the same endpoint while a handler runs owns that endpoint now: nothing the handler +// returns or throws for the old generation may reach it. +async function restartDuringHandler(settle: (release: { resolve: () => void; reject: (error: Error) => void }) => void) { + let release: { resolve: () => void; reject: (error: Error) => void } | undefined; + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: async (request) => { + if (request.messageId === "held") { + await new Promise((resolve, reject) => { release = { resolve, reject: (error) => reject(error) }; }); + } + return { + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }; + }, + }); + const connectionId = 0x43554c54; + const port = Number(new URL(server.endpoint).port); + const socket = await bindUdpSocket(); + let session = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const answered: string[] = []; + const goodbyes: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "disconnect") goodbyes.push(Buffer.from(packet.payload ?? []).toString("utf8")); + const result = session.receive(packet, Date.now()); + if (packet.packetType === "accept" || packet.reliable) { + socket.send(encodeRudpPacket(session.createAckForReceived(packet.sequence)), port, "127.0.0.1"); + } + for (const frame of result.delivered) { + answered.push((decodeMsgpack(frame.payload) as { messageId: string }).messageId); + } + }); + const toServer = (packet: CultNetRudpPacket) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1"); + const request = (messageId: string) => Buffer.from(encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + } satisfies CultNetOperationRequestMessage))); + try { + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the first Accept"); + toServer(session.send("schema", request("held"), { reliable: true, ordered: true })); + await waitFor(() => release !== undefined, "the held handler"); + + session = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the restarted client's Accept"); + settle(release!); + await new Promise((resolve) => setTimeout(resolve, 100)); + toServer(session.send("schema", request("after-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("after-restart"), "the restarted client's answer"); + + assert.deepEqual(answered, ["after-restart"], "the old generation's response reached the new session"); + assert.deepEqual(goodbyes, [], "a goodbye for the old generation reached the new session"); + assert.equal(session.connected, true); + } finally { + socket.close(); + await server.close(); + } +} + +test("an operation response for a generation a new Connect replaced is not sent into the new session", async () => { + await restartDuringHandler((release) => release.resolve()); +}); + +test("an operation handler that fails after a new Connect replaced its generation says no goodbye to the new session", async () => { + await restartDuringHandler((release) => release.reject(new Error("handler failed"))); +}); + test("RUDP operation service admits a restarted client on the same address", async () => { const server = await startCultNetOperationServer({ runtimeId: "sai-sidecar", diff --git a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts index 756ef97d..598d6fe6 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts @@ -61,13 +61,6 @@ const cultMeshTsRoot = resolve(cultLibRoot, "packages", "cultmesh-ts"); const cultMeshQuicPeerScript = resolve(cultMeshTsRoot, "dist-test", "test", "interop", "cultmesh-quic-peer.js"); const quicNativeTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Native.Tests", "GameCult.Mesh.Quic.Native.Tests.csproj"); const quicManagedTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Tests", "GameCult.Mesh.Quic.Tests.csproj"); -const rustBinaryPath = resolve( - cultnetRsRoot, - "target", - "debug", - "examples", - process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", -); const discoveryGroup = "239.77.44.11"; const serveReadyTimeoutMs = 90_000; @@ -78,6 +71,67 @@ let csharpInteropPeerBuild: Promise | undefined; let kotlinInteropPeerBuild: Promise | undefined; let cultMeshTsInteropPeerBuild: Promise | undefined; +let rustInteropPeerPathCache: string | undefined; +let tsInteropPeerChecked = false; +let pythonInteropChecked = false; + +// Cargo decides where it builds: CARGO_TARGET_DIR, a config's build.target-dir, or the crate's own +// target directory. The peer's path is asked of cargo, not assumed. +function rustInteropPeerPath(): string { + if (rustInteropPeerPathCache !== undefined) { + return rustInteropPeerPathCache; + } + const metadata = spawnSync(cargoCommand, ["metadata", "--format-version", "1", "--no-deps"], { + cwd: cultnetRsRoot, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (metadata.status !== 0) { + throw new Error( + `The Rust interop peer's target directory is unknown: \`cargo metadata\` failed in ${cultnetRsRoot}.\n${metadata.stderr ?? metadata.error?.message ?? ""}`, + ); + } + const targetDirectory = (JSON.parse(metadata.stdout) as { target_directory: string }).target_directory; + rustInteropPeerPathCache = resolve( + targetDirectory, + "debug", + "examples", + process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", + ); + return rustInteropPeerPathCache; +} + +// The TypeScript peer imports @gamecult/cultcache-ts from its built output. Unbuilt, the peer dies +// on import and a lane fails with a spawn or timeout error that names neither. +function tsInteropPeerScript(): string { + if (!tsInteropPeerChecked) { + try { + require.resolve("@gamecult/cultcache-ts"); + } catch { + throw new Error( + "@gamecult/cultcache-ts is not built, and the TypeScript interop peer imports it. Run `npm run build --workspace packages/cultcache-ts` first.", + ); + } + tsInteropPeerChecked = true; + } + return tsPeerScript; +} + +// The Python peers import msgpack. Missing, each Python lane fails with a traceback in a child's +// stderr instead of a reason. +function pythonInteropCommand(): string { + if (!pythonInteropChecked) { + const probe = spawnSync(pythonCommand, ["-c", "import msgpack"], { encoding: "utf8" }); + if (probe.status !== 0) { + throw new Error( + `${pythonCommand} cannot import msgpack, which the Python interop peers need. Install the Python packages first: \`python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py\`.\n${probe.stderr ?? probe.error?.message ?? ""}`, + ); + } + pythonInteropChecked = true; + } + return pythonCommand; +} + function resolvePythonCommand(): string { const codexPythonCommand = join(homedir(), ".cache", "codex-runtimes", "codex-primary-runtime", "dependencies", "python", "python.exe"); const candidates = process.platform === "win32" @@ -743,7 +797,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start rust peer"); servers.push(await spawnServeProcess("rust", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-peer", @@ -765,7 +819,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state servers.push(await spawnServeProcess("ts", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-peer", "--runtime-kind", "node", @@ -805,7 +859,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start python peer"); servers.push(await spawnServeProcess("python", { - command: pythonCommand, + command: pythonInteropCommand(), args: [ "-m", "cultnet_py.interop_peer", "serve", @@ -827,7 +881,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials rust"); const tsDial = await runJsonCommand("ts-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-client", "--runtime-kind", "node", @@ -846,7 +900,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials typescript over rudp"); const tsRudpDial = await runJsonCommand("ts-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rudp-client", "--runtime-kind", "node", @@ -864,7 +918,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "rust dials csharp"); - const rustDial = await runJsonCommand("rust-dial", rustBinaryPath, [ + const rustDial = await runJsonCommand("rust-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-client", "--runtime-kind", "rust", @@ -899,7 +953,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(csharpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials rust"); - const pythonDial = await runJsonCommand("python-dial", pythonCommand, [ + const pythonDial = await runJsonCommand("python-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-client", @@ -917,7 +971,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(pythonDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials python over rudp"); - const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonCommand, [ + const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-rudp-client", @@ -937,7 +991,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials python"); const tsDialPython = await runJsonCommand("ts-dial-python", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-python-client", "--runtime-kind", "node", @@ -1257,7 +1311,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript discovery probe"); await expectProbePeers("ts-probe", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "probe", "--runtime-id", "ts-prober", "--discovery-port", String(discoveryPort), @@ -1266,7 +1320,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultNetTsRoot, {}, expectedPeers); logInteropPhase("schema-v0", "rust discovery probe"); - await expectProbePeers("rust-probe", rustBinaryPath, [ + await expectProbePeers("rust-probe", rustInteropPeerPath(), [ "probe", "--runtime-id", "rust-prober", "--discovery-port", String(discoveryPort), @@ -1285,7 +1339,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultLibRoot, {}, expectedPeers); logInteropPhase("schema-v0", "python discovery probe"); - await expectProbePeers("python-probe", pythonCommand, [ + await expectProbePeers("python-probe", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "probe", "--runtime-id", "python-prober", @@ -1319,7 +1373,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD }); servers.push(await spawnServeProcess("rust-rudp", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-rudp-peer", @@ -1341,7 +1395,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD servers.push(await spawnServeProcess("ts-rust-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-rust-rudp-peer", "--runtime-kind", "node", @@ -1361,7 +1415,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD logInteropPhase("ts-rust-full-rudp", "typescript dials rust over rudp"); const tsRudpDial = await runJsonCommand("ts-rust-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rust-rudp-client", "--runtime-kind", "node", @@ -1379,7 +1433,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("ts-rust-full-rudp", "rust dials typescript over rudp"); - const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustBinaryPath, [ + const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-rudp-client", "--runtime-kind", "rust", @@ -2186,7 +2240,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" servers.push(await spawnServeProcess("ts-csharp-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-csharp-rudp-peer", "--runtime-kind", "node", @@ -2206,7 +2260,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" logInteropPhase("ts-csharp-full-rudp", "typescript dials csharp over rudp"); const tsRudpDial = await runJsonCommand("ts-csharp-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-csharp-rudp-client", "--runtime-kind", "node", @@ -2289,7 +2343,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R servers.push(await spawnServeProcess("ts-kotlin-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-kotlin-rudp-peer", "--runtime-kind", "node", @@ -2309,7 +2363,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R logInteropPhase("ts-kotlin-full-rudp", "typescript dials kotlin over rudp"); const tsRudpDial = await runJsonCommand("ts-kotlin-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-kotlin-rudp-client", "--runtime-kind", "node", @@ -2962,7 +3016,7 @@ interface RunningPythonRudpPeer { } function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpPeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpPeerScript], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3018,7 +3072,7 @@ function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpMessagePeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessagePeerScript], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3066,7 +3120,7 @@ function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { } function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-once", "--bind-host", "127.0.0.1", ...(options.clientPayload ? ["--client-payload", options.clientPayload] : []), @@ -3124,7 +3178,7 @@ function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnRustRudpMessageServer(): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-message-once", "--bind-host", "127.0.0.1", ], { @@ -3405,7 +3459,7 @@ function spawnKotlinRudpMessageServer(): RunningPythonRudpPeer { } function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3420,7 +3474,7 @@ function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpMessageClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessageClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3517,7 +3571,7 @@ function spawnCSharpRudpMessageClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3536,7 +3590,7 @@ function spawnRustRudpClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-message-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3832,7 +3886,7 @@ async function runPythonCultMeshClient(port: number): Promise { " 'peers': [{'peerId': peer.peer_id, 'roles': list(peer.roles)} for peer in peers],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3857,7 +3911,7 @@ async function runPythonCultNetRawClient(port: number): Promise { " 'shards': [{'shardId': shard.get('shardId'), 'epoch': shard.get('epoch')} for shard in shards.get('shards', [])],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3894,7 +3948,7 @@ async function runPythonCultMeshNodeSync(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3959,7 +4013,7 @@ async function runPythonCultMeshNodeEmit(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4013,7 +4067,7 @@ async function runPythonCultNetSubscription(port: number): Promise { " 'recordKey': change.get('document', {}).get('recordKey'),", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4067,7 +4121,7 @@ async function runPythonCultMeshPredictionReconcile(port: number): Promise " 'note': note,", " }))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts new file mode 100644 index 00000000..4370d406 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -0,0 +1,520 @@ +// A datagram that cannot be sent to one peer is that peer's problem alone. Node throws +// for a send it refuses outright (a peer at UDP port 0, which any spoofer can claim) +// and reports a system failure later through the callback; neither may escape a +// timer or a receive handler, and only a datagram that can never be sent as built +// ends the session that owes it. +import assert from "node:assert/strict"; +import dgram, { type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode } from "@msgpack/msgpack"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + decodeRudpPacket, + encodeCultNetMessageForWire, + encodeRudpPacket, + invokeCultNetOperation, + isPermanentSendError, + startCultNetOperationServer, + type CultNetOperationRequestMessage, + type CultNetOperationResponseMessage, +} from "../src"; + +async function bindUdpSocket(): Promise { + const socket = dgram.createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + return socket; +} + +function udpPort(socket: Socket): number { + const address = socket.address(); + assert.notEqual(typeof address, "string"); + return address.port; +} + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 2_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** A Connect as it arrives from UDP source port 0: Node refuses to send anything back. */ +function connectFromPortZero(socket: Socket, connectionId: number): void { + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); +} + +/** Records every exception that would otherwise end the process. */ +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +test("only a datagram that can never be sent as built is a permanent send failure", () => { + const failure = (code: string) => Object.assign(new Error("send failed"), { code }); + for (const code of ["EMSGSIZE", "EINVAL", "EAFNOSUPPORT", "ERR_SOCKET_BAD_PORT"]) { + assert.equal(isPermanentSendError(failure(code)), true, code); + } + for (const code of ["EHOSTUNREACH", "ENETUNREACH", "ECONNREFUSED", "ENOBUFS", "EAGAIN", "ERR_SOCKET_DGRAM_NOT_RUNNING"]) { + assert.equal(isPermanentSendError(failure(code)), false, code); + } + assert.equal(isPermanentSendError(new Error("no code")), false); +}); + +test("a server transport answering a Connect from port 0 ends that session and keeps serving", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203090; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: Error[] = []; + const reasons: string[] = []; + server.on("error", (error: Error) => errors.push(error)); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + connectFromPortZero(serverSocket, connectionId); + // Many resend ticks: nothing the port-0 session owed is ever retried. + await sleep(100); + // The goodbye names the error by its code, never by a message quoting the address. + assert.deepEqual(reasons, ["packet could not be sent: ERR_SOCKET_BAD_PORT"]); + + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + client.connect(); + await waitFor(() => client.connected && server.connected, "a real client's handshake"); + client.send("schema", Buffer.from("after", "utf8")); + await waitFor(() => frames.length === 1, "the real client's frame"); + assert.deepEqual(frames, ["after"]); + assert.deepEqual(errors, []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); + +test("an operation server answering a Connect from port 0 starts no session and keeps serving", async () => { + const connectionId = 0x43554c54; + const uncaught = watchUncaught(); + // The server owns its socket; this captures it so a datagram can arrive from port 0. + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const raw = await bindUdpSocket(); + const port = Number(new URL(server.endpoint).port); + const accepts: number[] = []; + raw.on("message", () => accepts.push(Date.now())); + try { + connectFromPortZero(serverSocket!, connectionId); + // The resend timer runs every 25 ms; a session that had started would owe its Accept. + await sleep(150); + assert.equal(server.packetsDropped, 1, "the Connect started no session"); + assert.equal(server.sendFailures, 0); + + // A real peer is still answered. + raw.send(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(0)), port, "127.0.0.1"); + await waitFor(() => accepts.length > 0, "the real peer's Accept"); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + raw.close(); + await server.close(); + } +}); + +test("a server transport whose resend can never be sent ends the session and sends nothing more of it", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203094; + const uncaught = watchUncaught(); + const clientPort = udpPort(clientSocket); + // Sends to the client fail as the test says: later through the callback, or at once. + let mode: "healthy" | "lost" | "never" = "healthy"; + const send = serverSocket.send.bind(serverSocket) as (...args: unknown[]) => void; + Object.assign(serverSocket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + if (port !== clientPort || mode === "healthy") { + send(message, port, address, callback); + } else if (mode === "lost") { + process.nextTick(() => callback?.(Object.assign(new Error("lost"), { code: "EHOSTUNREACH" }))); + } else { + throw Object.assign(new Error("never"), { code: "EMSGSIZE" }); + } + }, + }); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: string[] = []; + const reasons: string[] = []; + server.on("error", (error: NodeJS.ErrnoException) => errors.push(error.code ?? "")); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + mode = "lost"; + server.send("schema", Buffer.from("one", "utf8")); + server.send("schema", Buffer.from("two", "utf8")); + await waitFor(() => server.stats.sendFailures >= 4, "both writes lost, and lost again when resent"); + assert.deepEqual(errors, [], "a loss that may pass is counted, never emitted"); + assert.deepEqual(reasons, []); + + // Both writes are due again together; the first can never be sent. + mode = "never"; + await waitFor(() => reasons.length > 0, "the session to end"); + await sleep(50); + assert.deepEqual(reasons, ["packet could not be sent: EMSGSIZE"]); + assert.equal(server.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); + +// The kernel refuses a datagram to the limited broadcast address from a socket that has +// not asked for broadcast (EACCES): a real failure that may pass once the socket or the +// route changes, so it is a lost datagram. Node reports it only to the send's callback, +// never on the socket's "error" event, so nothing may turn it into an "error" either. +const UNREACHABLE = { host: "255.255.255.255", port: 9 }; + +test("a client transport with no error listener counts a send the network refuses and does not end the process", async () => { + const socket = await new Promise((resolve) => { + const created = dgram.createSocket("udp4"); + created.bind(0, "0.0.0.0", () => resolve(created)); + }); + const uncaught = watchUncaught(); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket, + mode: "client", + remoteHost: UNREACHABLE.host, + remotePort: UNREACHABLE.port, + connectionId: 0x10203095, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + // The Connect and its resends, each refused by the kernel. + await waitFor(() => client.stats.sendFailures >= 5, "the refused Connect and its resends"); + assert.equal(client.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + } +}); + +test("an operation call the network refuses rejects with a timeout and does not end the process", async () => { + const uncaught = watchUncaught(); + try { + await assert.rejects( + invokeCultNetOperation( + `rudp://${UNREACHABLE.host}:${UNREACHABLE.port}`, + { + schemaVersion: "cultnet.operation_request.v0", + messageId: "refused", + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + sourceRuntimeId: "caller", + }, + { runtimeId: "caller", timeoutMs: 400 }, + ), + /CultNet operation connection timed out/, + ); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + } +}); + +// A send's failure can be reported after the generation that sent it ended: the client +// connected again from the same endpoint in the meantime. That failure is owed by nothing; +// it may not end the generation that replaced it or say goodbye to the endpoint. + +/** Makes the next send to `port` fail as can never pass as built, reported `delayMs` later. */ +function failNextSendLate(socket: Socket, port: number, delayMs: number): { arm: () => void } { + let armed = false; + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, toPort: number, address: string, callback?: (error: Error | null) => void) => { + if (toPort !== port || !armed) { + send(message, toPort, address, callback); + return; + } + armed = false; + setTimeout(() => callback?.(Object.assign(new Error("late"), { code: "EMSGSIZE" })), delayMs); + }, + }); + return { arm: () => { armed = true; } }; +} + +/** A client driven packet by packet from one socket, so it can connect again from the same endpoint. */ +async function rawClient(serverPort: number, connectionId: number) { + const socket = await bindUdpSocket(); + const received: ReturnType[] = []; + socket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const toServer = (packet: Parameters[0]) => socket.send(encodeRudpPacket(packet), serverPort, "127.0.0.1"); + const connect = async (initialSequence: number) => { + const session = new CultNetRudpSession({ connectionId, initialSequence }); + received.length = 0; + toServer(session.createConnect(Date.now())); + await waitFor(() => received.some((packet) => packet.packetType === "accept"), "the Accept"); + const accept = received.find((packet) => packet.packetType === "accept")!; + session.receive(accept, Date.now()); + toServer(session.createAckForReceived(accept.sequence)); + return session; + }; + return { socket, received, toServer, connect }; +} + +test("a transport ignores a send failure reported after the generation that sent it ended", async () => { + const serverSocket = await bindUdpSocket(); + const connectionId = 0x10203096; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1_000, + }); + const reasons: string[] = []; + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = await rawClient(udpPort(serverSocket), connectionId); + const late = failNextSendLate(serverSocket, udpPort(client.socket), 150); + try { + await client.connect(10); + late.arm(); + server.send("schema", Buffer.from("old", "utf8")); + await client.connect(5_000); + await sleep(250); + + assert.deepEqual(reasons, []); + assert.equal(server.connected, true); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.socket.close(); + server.close(); + } +}); + +test("an operation server ignores a send failure reported after the generation that sent it ended", async () => { + const connectionId = 0x43554c54; + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const client = await rawClient(Number(new URL(server.endpoint).port), connectionId); + const late = failNextSendLate(serverSocket!, udpPort(client.socket), 150); + const request = (session: CultNetRudpSession, messageId: string) => session.sendMany("schema", encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + sourceRuntimeId: "caller", + } satisfies CultNetOperationRequestMessage, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); + try { + const old = await client.connect(10); + // The response to this request is the next send, and it fails after the client reconnected. + late.arm(); + for (const packet of request(old, "old")) client.toServer(packet); + const renewed = await client.connect(5_000); + await sleep(250); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.equal(server.sendFailures, 0); + + // The session that replaced it is still served. + let read = client.received.length; + let answered = 0; + for (const packet of request(renewed, "after")) client.toServer(packet); + await waitFor(() => { + for (; read < client.received.length; read += 1) { + const packet = client.received[read]!; + if (packet.packetType === "data") answered += renewed.receive(packet, Date.now()).delivered.length; + } + return answered > 0; + }, "the renewed session's response"); + } finally { + client.socket.close(); + await server.close(); + } +}); + +test("a caller's send larger than a UDP datagram throws and takes nothing from the session", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203097; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // 36 header bytes and the six of "schema": the largest payload one IPv4 datagram carries. + const largest = 65_507 - 36 - "schema".length; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + await waitFor(() => client.outstandingReliablePacketCount === 0, "the Connect acknowledged"); + + client.send("schema", new Uint8Array(largest)); + assert.throws(() => client.send("schema", new Uint8Array(largest + 1)), { code: "EMSGSIZE" }); + assert.equal(client.stats.framesSent, 1); + client.send("schema", Buffer.from("next", "utf8")); + + // The refused send took no sequence: the next frame is not held behind a gap. + await waitFor(() => frames.length === 2, "both sent frames"); + assert.deepEqual(frames, [largest, 4]); + assert.equal(client.connected, true); + await waitFor(() => client.outstandingReliablePacketCount === 0, "both frames acknowledged"); + } finally { + client.close(); + server.close(); + } +}); + +// The pre-check measures one fragment, the largest datagram a fragmented send makes. Fragments +// of 1 KiB and of 40,000 bytes both leave a 70,000-byte send legal; the second size is large +// enough that any small multiple of it would not be. +for (const maxFragmentBytes of [1024, 40_000]) { + test(`a fragmented send larger than one UDP datagram is sent, because each datagram carries one fragment (${maxFragmentBytes}-byte fragments)`, async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203095; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + maxFragmentBytes, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // Larger than any one datagram can carry, over IPv4 or IPv6. + const payloadBytes = 70_000; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + + assert.doesNotThrow(() => client.send("schema", new Uint8Array(payloadBytes))); + assert.equal(client.stats.framesSent, 1); + + await waitFor(() => frames.length === 1, "the reassembled frame"); + assert.deepEqual(frames, [payloadBytes]); + assert.equal(client.connected, true); + } finally { + client.close(); + server.close(); + } + }); +} diff --git a/samples/eve-browser-network/Program.cs b/samples/eve-browser-network/Program.cs index c26d654b..a8cc3429 100644 --- a/samples/eve-browser-network/Program.cs +++ b/samples/eve-browser-network/Program.cs @@ -244,11 +244,13 @@ static async Task RunHeadlessAsync(Args arguments) Console.WriteLine("HEADLESS_READY " + JsonSerializer.Serialize(new { count = initial.Count })); } if (initial.Count >= arguments.ExpectedCount) completion.TrySetResult(); + var commandReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var commandTask = Task.Run(async () => { var line = await Console.In.ReadLineAsync(); if (line == null || !line.StartsWith("INVOKE ", StringComparison.Ordinal)) throw new InvalidOperationException("The headless sample expected 'INVOKE ' on stdin."); + commandReceived.TrySetResult(); var commandId = line["INVOKE ".Length..].Trim(); var result = await mesh.InvokeAsync( target, @@ -268,6 +270,9 @@ static async Task RunHeadlessAsync(Args arguments) Console.WriteLine("HEADLESS_NETWORK_BENCHMARK " + JsonSerializer.Serialize( await MeasureOperationSessionAsync(mesh, target, arguments))); }); + // How long the caller takes to send the command is the caller's to bound + // (closing stdin ends the wait); this process bounds only its own work. + await Task.WhenAny(commandReceived.Task, commandTask); await Task.WhenAll(completion.Task, commandTask).WaitAsync(TimeSpan.FromSeconds(45)); } diff --git a/scripts/pack-nuget.ps1 b/scripts/pack-nuget.ps1 index 95e5cd2f..46e0192c 100644 --- a/scripts/pack-nuget.ps1 +++ b/scripts/pack-nuget.ps1 @@ -13,55 +13,90 @@ $outputRoot = if ([IO.Path]::IsPathRooted($OutputDirectory)) { Join-Path $repoRoot $OutputDirectory } -$projects = @( - "src\GameCult.Logging\GameCult.Logging.csproj", - "src\GameCult.Caching\GameCult.Caching.csproj", - "src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj", - "src\GameCult.Networking\GameCult.Networking.csproj", - "src\GameCult.Networking.WebSockets\GameCult.Networking.WebSockets.csproj", - "src\GameCult.Mesh\GameCult.Mesh.csproj" +# The public packages a consumer references directly. Everything they reach +# through ProjectReference is part of the published closure and is packed too, +# so a new internal dependency cannot leave the feed short of a package. +$rootProjects = @( + "src\GameCult.Mesh\GameCult.Mesh.csproj", + "src\GameCult.Networking.WebSockets\GameCult.Networking.WebSockets.csproj" ) +# MSBuild paths use backslashes; normalize them so the closure walk also +# resolves on Linux and macOS. +function Resolve-ProjectPath([string] $path) { + return [IO.Path]::GetFullPath($path.Replace('\', [string][IO.Path]::DirectorySeparatorChar)) +} + +function Get-ProjectReferenceClosure([string[]] $roots) { + $seen = [System.Collections.Generic.List[string]]::new() + $pending = [System.Collections.Generic.Queue[string]]::new() + foreach ($root in $roots) { $pending.Enqueue((Resolve-ProjectPath $root)) } + while ($pending.Count -gt 0) { + $project = $pending.Dequeue() + if ($seen.Contains($project)) { continue } + if (-not (Test-Path -LiteralPath $project -PathType Leaf)) { + throw "Project in the NuGet closure was not found: $project" + } + $seen.Add($project) + # Every ProjectReference counts, whatever its Condition: a reference + # present for any target framework ships in the package's dependencies. + foreach ($reference in ([xml](Get-Content -LiteralPath $project -Raw)).SelectNodes("//*[local-name()='ProjectReference']")) { + $include = $reference.GetAttribute("Include").Trim() + if ($include -match '\$\(') { + throw "Cannot resolve the MSBuild property in ProjectReference '$include' from $project" + } + $pending.Enqueue((Resolve-ProjectPath (Join-Path (Split-Path -Parent $project) $include))) + } + } + return $seen +} + +$projects = Get-ProjectReferenceClosure ($rootProjects | ForEach-Object { Join-Path $repoRoot $_ }) + if (Test-Path -LiteralPath $outputRoot) { Remove-Item -LiteralPath $outputRoot -Recurse -Force } New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null +$versionArguments = @() +if (-not [string]::IsNullOrWhiteSpace($PackageVersion)) { + $versionArguments += "-p:CultLibPackageVersion=$PackageVersion" +} + +$expectedPackages = @() foreach ($project in $projects) { - $packArguments = @( - "pack", (Join-Path $repoRoot $project), "-c", $Configuration, "-o", $outputRoot, - "--nologo", "--verbosity", "quiet", "-p:NoWarn=1591%3BCS8632" - ) - if (-not [string]::IsNullOrWhiteSpace($PackageVersion)) { - $packArguments += "-p:CultLibPackageVersion=$PackageVersion" + # The package identity comes from MSBuild itself: GameCult.Math, for one, + # carries its own PackageId and version rather than the CultLib version. + $identity = (& dotnet msbuild $project -nologo "-getProperty:PackageId" "-getProperty:Version" @versionArguments) | + Out-String | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { + throw "Could not read the package identity of $project" } + $expectedPackages += "$($identity.Properties.PackageId).$($identity.Properties.Version).nupkg" + + $packArguments = @( + "pack", $project, "-c", $Configuration, "-o", $outputRoot, + "--nologo", "--verbosity", "quiet", "-p:NoWarn=1591%3BCS8632" + ) + $versionArguments & dotnet @packArguments if ($LASTEXITCODE -ne 0) { throw "NuGet pack failed for $project with exit code $LASTEXITCODE" } } +foreach ($package in $expectedPackages) { + if (-not (Test-Path -LiteralPath (Join-Path $outputRoot $package))) { + throw "NuGet dependency closure is missing $package" + } +} + $meshPackage = Get-ChildItem -LiteralPath $outputRoot -Filter "GameCult.Mesh.*.nupkg" | Where-Object { $_.Name -notlike "*.snupkg" } | Select-Object -First 1 if (-not $meshPackage) { throw "GameCult.Mesh package was not produced." } - $version = [regex]::Match($meshPackage.Name, '^GameCult\.Mesh\.(.+)\.nupkg$').Groups[1].Value -$expectedPackages = @( - "GameCult.Logging.$version.nupkg", - "GameCult.Caching.$version.nupkg", - "GameCult.Caching.MessagePack.$version.nupkg", - "GameCult.Networking.$version.nupkg", - "GameCult.Networking.WebSockets.$version.nupkg", - "GameCult.Mesh.$version.nupkg" -) -foreach ($package in $expectedPackages) { - if (-not (Test-Path -LiteralPath (Join-Path $outputRoot $package))) { - throw "NuGet dependency closure is missing $package" - } -} if (-not $SkipConsumerSmoke) { $smokeRoot = Join-Path $repoRoot "artifacts\nuget-consumer-smoke" diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 76656699..0d7ebd16 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -537,8 +537,12 @@ namespace GameCult.Mesh /// public sealed class CultMeshRudpSocketOptions { - ///

Gets or sets the local bind host. - public string BindHost { get; set; } = "0.0.0.0"; + /// + /// Gets or sets the local bind host. Unset, a server binds every IPv4 interface, and a + /// client binds loopback for a loopback endpoint and the unspecified address of the + /// endpoint's family otherwise, so it can reach a remote host. + /// + public string? BindHost { get; set; } /// Gets or sets the local bind port. public int BindPort { get; set; } /// Gets or sets a caller-owned bound socket. @@ -2933,7 +2937,7 @@ namespace GameCult.Mesh return new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions { RuntimeId = runtimeId, - Socket = options.Socket ?? BindRudpSocket(options.BindHost, options.BindPort), + Socket = options.Socket ?? BindRudpSocket(ResolveRudpAddress(options.BindHost ?? "0.0.0.0"), options.BindPort), Mode = CultNetRudpSocketMode.Server, ConnectionId = connectionId, InitialSequence = options.InitialSequence, @@ -2957,12 +2961,16 @@ namespace GameCult.Mesh { if (endpoint == null) throw new ArgumentNullException(nameof(endpoint)); options ??= new CultMeshRudpSocketOptions(); + var remote = ResolveRudpAddress(endpoint.Host); + var bindAddress = options.BindHost == null + ? RudpClientBindAddress(remote) + : ResolveRudpAddress(options.BindHost); return new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions { RuntimeId = runtimeId, - Socket = options.Socket ?? BindRudpSocket(options.BindHost, options.BindPort), + Socket = options.Socket ?? BindRudpSocket(bindAddress, options.BindPort), Mode = CultNetRudpSocketMode.Client, - RemoteEndPoint = new IPEndPoint(ResolveRudpAddress(endpoint.Host), endpoint.Port), + RemoteEndPoint = new IPEndPoint(remote, endpoint.Port), ConnectionId = connectionId, InitialSequence = options.InitialSequence, ResendDelayMs = options.ResendDelayMs, @@ -3234,9 +3242,21 @@ namespace GameCult.Mesh return CultNetLocal.ConnectClient(host, port, security, configureClient); } - private static Socket BindRudpSocket(string host, int port) + // A socket bound to loopback cannot send off the host on Windows, so a client keeps + // loopback only when its peer is on loopback. + private static IPAddress RudpClientBindAddress(IPAddress remote) + { + var ipv6 = remote.AddressFamily == AddressFamily.InterNetworkV6; + if (IPAddress.IsLoopback(remote)) + { + return ipv6 ? IPAddress.IPv6Loopback : IPAddress.Loopback; + } + + return ipv6 ? IPAddress.IPv6Any : IPAddress.Any; + } + + private static Socket BindRudpSocket(IPAddress address, int port) { - var address = ResolveRudpAddress(host); var socket = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); socket.Bind(new IPEndPoint(address, port)); socket.ReceiveTimeout = 20; diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index e19fccc3..de8d0154 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Text; using System.Threading.Tasks; using GameCult.Logging; @@ -169,6 +170,9 @@ namespace GameCult.Networking /// public CultNetTransportStats Stats => _transport.Stats; + // The listener this server drives, for tests that inject send failures through its seams. + internal CultNetRudpSocketTransportServer Transport => _transport; + /// /// Gets the currently tracked RUDP peers. /// @@ -283,26 +287,47 @@ namespace GameCult.Networking return new RudpCultNetSchemaPollResult(consumed, dispatched); } + /// + /// The goodbye's reason when a handler fails on a peer's message: the server could not handle + /// it, which says nothing about whether the peer's packet was well formed. + /// + internal static readonly byte[] HandlerFailedReason = Encoding.UTF8.GetBytes("server could not handle a message"); + + private static string SessionOutcome(bool ended) => + ended ? "that peer's session ended." : "that peer's session had already ended."; + private async Task DispatchAsync(CultNetRudpSocketServerFrame delivered) { if (!string.Equals(delivered.Frame.ChannelId, "schema", StringComparison.Ordinal)) return false; - var message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + // A peer's message that cannot be decoded (malformed bytes, an unknown schemaVersion) or + // handled ends that peer's session, never the poll: the drain goes on serving every other + // peer. The transport already acknowledged the frame, so the session cannot be kept (a + // retransmit would never be handled). The peer is told, and the drop is counted and logged. + // The C# TCP schema server (CultNetTcpSchemaTransport.ServePeerAsync) likewise ends the one + // peer whose message fails to decode or handle. A handler is still expected to answer its + // own typed refusals on the wire (HandleSnapshotRequestV1Async does, R-AM); this is the + // backstop for whatever it does not. The log names the fault's type only: its message can + // quote what the peer sent. + ICultNetSchemaMessage message; + try + { + message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + } + catch (Exception ex) + { + // The peer sent something no runtime can read: the refusal is the peer's. + var ended = _transport.EndPeerSession(delivered.Peer, CultNetRudpSession.RefusedPacketReason); + Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); {SessionOutcome(ended)}"); + return false; + } + if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) { return false; } - // R-AG: a handler is expected to catch its own typed refusals (HandleSnapshotRequestV1Async - // catches the three selection exceptions and answers cursor_invalid/selection_invalid/ - // reference_outside_target on the wire, R-AM) - this catch is the backstop for whatever a - // handler does not, so an untyped exception from a malformed/hostile message can never - // reach the poll loop and stall the server for every other peer. It still answers nothing - // on the wire (an exception this generic carries no typed refusal to report) and drops the - // one malformed dispatch, exactly as an unregistered handler already does just above - - // R-AM: but it always logs. A backstop that swallows silently is a compensator, and an - // untested, unlogged one hid the SM-6 ArgumentNullException regression for a whole batch. try { var result = handler.DynamicInvoke(message, new RudpCultNetSchemaServerPeer(this, delivered.Peer)); @@ -313,11 +338,14 @@ namespace GameCult.Networking } catch (Exception ex) { - // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException, - // whose own .Message is the generic "Exception has been thrown by the target of an - // invocation." - log the real fault, not that wrapper. + // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException; + // name the real fault, not that wrapper. The fault is the server's (a response too large + // to send, a handler bug) or a refusal the handler did not type, so the goodbye does not + // say the peer's packet was refused. A response that failed after its first fragment + // already ended the session over the send, and the log says so. var reported = ex is System.Reflection.TargetInvocationException { InnerException: { } inner } ? inner : ex; - Logger.LogError($"CultNet RUDP schema dispatch failed for {message.GetType().Name}: {reported.Message}"); + var ended = _transport.EndPeerSession(delivered.Peer, HandlerFailedReason); + Logger.LogError($"CultNet RUDP schema server's {message.GetType().Name} handler failed ({reported.GetType().Name}); {SessionOutcome(ended)}"); return false; } return true; diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 9799cb38..a6102de3 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1,5 +1,6 @@ using System; using System.Buffers.Binary; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.IO; @@ -39,6 +40,12 @@ namespace GameCult.Networking /// the peer, or a packet the session refuses. /// public long PacketsDropped { get; internal set; } + /// + /// Gets the number of datagrams a multi-peer server could not send to one peer. Each is + /// that peer's lost datagram: a reliable packet stays pending and is resent, and no other + /// peer is affected. Zero for single-peer transports, whose callers get the send error. + /// + public long SendFailures { get; internal set; } internal CultNetTransportStats Snapshot() { @@ -48,7 +55,8 @@ namespace GameCult.Networking FramesReceived = FramesReceived, BytesSent = BytesSent, BytesReceived = BytesReceived, - PacketsDropped = PacketsDropped + PacketsDropped = PacketsDropped, + SendFailures = SendFailures }; } } @@ -885,6 +893,10 @@ namespace GameCult.Networking { /// Maximum reliable packets admitted to the wire before acknowledgements advance the window. public const int ReliableSendWindowPackets = 32; + // Flow window: a reliable packet is admitted only while its sequence is at most FlowWindowSequences above the + // lowest unacked one and the payload above that sequence stays within FlowWindowBytes. + private const uint FlowWindowSequences = 1023; + private const int FlowWindowBytes = 4 * 1024 * 1024; private const int ReceivedSequenceWindow = 4096; // How long a client keeps retransmitting a Connect nobody answered before it abandons that // attempt for a fresh one. A Connect the server answers with an Ack, never an Accept, is one @@ -935,6 +947,9 @@ namespace GameCult.Networking private readonly Dictionary _latestSequencedByChannel = new Dictionary(StringComparer.Ordinal); private readonly object _pendingReliableGate = new object(); private readonly Dictionary _pendingReliable = new Dictionary(); + // Payload sizes of acknowledged reliable sequences above the lowest unacknowledged one. The receiver still + // holds those bytes behind the gap, so the flow window keeps counting them until the lowest sequence passes them. + private readonly Dictionary _ackedAboveLowest = new Dictionary(); private readonly Queue _queuedReliable = new Queue(); // Every reliable sequence up to and including this one has been received since the peer state // was last reset. Only the handshake seeds it: the peer's Connect on the accepting side, the @@ -1156,7 +1171,7 @@ namespace GameCult.Networking options ??= new CultNetRudpSendOptions(); lock (_pendingReliableGate) { - if (options.Reliable && _pendingReliable.Count >= ReliableSendWindowPackets) + if (options.Reliable && (_queuedReliable.Count > 0 || !WindowAdmits(_nextSequence, payload.Length))) throw new InvalidOperationException("RUDP reliable send window is full; receive acknowledgements before sending."); } return SendMany(channelId, payload, options).First(); @@ -1420,6 +1435,7 @@ namespace GameCult.Networking lock (_pendingReliableGate) { _pendingReliable.Clear(); + _ackedAboveLowest.Clear(); _queuedReliable.Clear(); } } @@ -1446,10 +1462,64 @@ namespace GameCult.Networking /// sequence, so the peer's sequences are forgotten before the goodbye is built: its ack field /// would otherwise acknowledge the very frame the session refused. /// - internal CultNetRudpPacket EndRefused() + internal CultNetRudpPacket EndRefused() => EndWith(RefusedPacketReason); + + /// + /// Ends a session whose peer's frame the application above it refused or failed on, after the + /// session acknowledged it. As with , the peer's sequences are forgotten + /// before the goodbye is built. + /// + internal CultNetRudpPacket EndWith(byte[] reason) { ResetPeerState(); - return CreateDisconnect(RefusedPacketReason); + return CreateDisconnect(reason); + } + + /// + /// Whether a failed send can never succeed for the datagram as built, so resending it is + /// pointless: it is too large for a datagram (EMSGSIZE, WSAEMSGSIZE), its address is + /// malformed for the socket (EINVAL, WSAEINVAL), or belongs to another address family + /// (EAFNOSUPPORT, WSAEAFNOSUPPORT). Everything else (no route, full buffers, firewall drops, + /// a refusal reported for an earlier datagram, an interrupted call) is a property of the path + /// or the moment and may pass, so it is a lost datagram, not a reason to end a session. + /// + internal static bool IsPermanentSendError(SocketException error) => + error.SocketErrorCode == SocketError.MessageSize || + error.SocketErrorCode == SocketError.InvalidArgument || + error.SocketErrorCode == SocketError.AddressFamilyNotSupported; + + internal const string UnsendablePacketReason = "packet could not be sent"; + + /// + /// Ends a session that owes its peer a packet that can never be sent as built. Resending it + /// would fail forever, and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. The goodbye's reason names the error. + /// + internal CultNetRudpPacket EndUnsendable(SocketException error) + { + ResetPeerState(); + return CreateDisconnect(Encoding.UTF8.GetBytes(UnsendablePacketReason + ": " + error.Message)); + } + + internal uint NextSequence => _nextSequence; + + /// + /// Undoes a send that never reached the wire: every packet it created at or after + /// , pending or queued, leaves no trace and its sequences are + /// reissued, so the peer never sees a gap. Only valid while nothing else has been sent since. + /// + internal void WithdrawSend(uint firstSequence) + { + lock (_pendingReliableGate) + { + foreach (var sequence in _pendingReliable.Keys.Where(sequence => sequence >= firstSequence).ToArray()) + _pendingReliable.Remove(sequence); + var kept = _queuedReliable.Where(packet => packet.Sequence < firstSequence).ToArray(); + _queuedReliable.Clear(); + foreach (var packet in kept) + _queuedReliable.Enqueue(packet); + } + _nextSequence = firstSequence; } /// @@ -1573,17 +1643,39 @@ namespace GameCult.Networking } } + /// + /// Whether a reliable packet may go on the wire now: the window has a slot, and its sequence and the + /// payload above the lowest unacked sequence stay inside the flow window. With nothing pending, any + /// packet is admissible. Callers hold the pending gate. + /// + private bool WindowAdmits(uint sequence, int payloadLength) + { + if (_pendingReliable.Count >= ReliableSendWindowPackets) + return false; + if (_pendingReliable.Count == 0) + return true; + var lowest = _pendingReliable.Keys.Min(); + long bytesAbove = 0; + foreach (var pending in _pendingReliable) + { + if (pending.Key > lowest) + bytesAbove += pending.Value.Packet.Payload.Length; + } + foreach (var length in _ackedAboveLowest.Values) + bytesAbove += length; + return sequence - lowest <= FlowWindowSequences && bytesAbove + payloadLength <= FlowWindowBytes; + } + private IReadOnlyList AdmitReliablePackets( IReadOnlyList packets, long nowMs) { lock (_pendingReliableGate) { - var available = Math.Max(0, ReliableSendWindowPackets - _pendingReliable.Count); - var ready = new List(Math.Min(available, packets.Count)); + var ready = new List(); foreach (var packet in packets) { - if (ready.Count < available) + if (_queuedReliable.Count == 0 && WindowAdmits(packet.Sequence, packet.Payload.Length)) { _pendingReliable[packet.Sequence] = new PendingReliablePacket { @@ -1605,9 +1697,9 @@ namespace GameCult.Networking { lock (_pendingReliableGate) { - var available = Math.Max(0, ReliableSendWindowPackets - _pendingReliable.Count); - var ready = new List(Math.Min(available, _queuedReliable.Count)); - while (ready.Count < available && _queuedReliable.Count > 0) + var ready = new List(); + while (_queuedReliable.Count > 0 + && WindowAdmits(_queuedReliable.Peek().Sequence, _queuedReliable.Peek().Payload.Length)) { var packet = _queuedReliable.Dequeue(); _pendingReliable[packet.Sequence] = new PendingReliablePacket @@ -1641,17 +1733,36 @@ namespace GameCult.Networking { lock (_pendingReliableGate) { - _pendingReliable.Remove(packet.Ack); + Acknowledge(packet.Ack); for (var bit = 0; bit < 32; bit++) { if ((packet.AckMask & (1u << bit)) != 0 && packet.Ack > bit) { - _pendingReliable.Remove(packet.Ack - (uint)bit - 1); + Acknowledge(packet.Ack - (uint)bit - 1); } } + // Acknowledged sizes stop counting once the lowest unacknowledged sequence passes them, and all + // of them stop when nothing is unacknowledged. + if (_pendingReliable.Count == 0) + { + _ackedAboveLowest.Clear(); + } + else + { + var lowest = _pendingReliable.Keys.Min(); + foreach (var sequence in _ackedAboveLowest.Keys.Where(sequence => sequence <= lowest).ToArray()) + _ackedAboveLowest.Remove(sequence); + } } } + // Callers hold the pending gate. + private void Acknowledge(uint sequence) + { + if (_pendingReliable.Remove(sequence, out var pending)) + _ackedAboveLowest[sequence] = pending.Packet.Payload.Length; + } + private static bool PacketAcknowledges(CultNetRudpPacket packet, uint sequence) { if (packet.Ack == sequence) @@ -2628,6 +2739,11 @@ namespace GameCult.Networking /// Gets the last transport-level remote disconnect reason, if one was received. /// public byte[]? DisconnectReason { get; internal set; } + + // The goodbye's reason once the session ended over a packet it could never send. Written + // under SessionGate by whichever thread's send found the failure; the polling thread reads + // it when it retires the peer. + internal byte[]? UnsendableReason { get; set; } } /// @@ -2665,6 +2781,10 @@ namespace GameCult.Networking private readonly CultNetTransportStats _stats = new CultNetTransportStats(); private readonly Dictionary _peers = new Dictionary(StringComparer.Ordinal); private readonly Queue _deliveredFrames = new Queue(); + // Peers whose session ended over a packet it could never send, waiting for the polling thread + // to retire them. Only the polling thread changes _peers or raises PeerDisconnected; a send on + // a caller's thread ends the session and leaves the peer here. + private readonly ConcurrentQueue _unsendablePeers = new ConcurrentQueue(); private bool _disposed; /// @@ -2726,8 +2846,28 @@ namespace GameCult.Networking public void Send(CultNetRudpSocketServerPeer peer, string channelId, byte[] payload) { if (peer == null) throw new ArgumentNullException(nameof(peer)); + SocketException? unsendable = null; + var failedIndex = 0; + long generation; lock (peer.SessionGate) - SendPackets(peer, peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes)); + { + var firstSequence = peer.Session.NextSequence; + var packets = peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes); + generation = peer.Session.Generation; + unsendable = SendPackets(peer, packets, out failedIndex); + // A caller-directed send that can never succeed is the caller's error and queues + // nothing. Only when earlier fragments are already on the wire has the peer seen + // part of it, and then the session cannot be kept. + if (unsendable != null && failedIndex == 0) + peer.Session.WithdrawSend(firstSequence); + } + if (unsendable != null) + { + // The polling thread retires the peer; this thread only ends its session. + if (failedIndex > 0) + EndUnsendableSession(peer, unsendable, generation); + throw unsendable; + } _stats.FramesSent++; } @@ -2772,6 +2912,7 @@ namespace GameCult.Networking /// public bool TryReceiveOnce(out CultNetRudpSocketServerFrame? delivered) { + RetireUnsendablePeers(); if (_deliveredFrames.Count > 0) { delivered = _deliveredFrames.Dequeue(); @@ -2834,13 +2975,24 @@ namespace GameCult.Networking _peers.TryGetValue(peerKey, out var admitted); // The admitted peer's session decides whether this Connect repeats the one it accepted: // answer a repeat with the Accept already owed and queue nothing. Any other Connect is - // a new generation. + // a new generation, and the admitted peer's session ends here, under the gate every + // Send takes: a caller still holding the replaced peer can send nothing more into the + // endpoint the new session now owns. if (admitted != null) { + var repeats = false; + SocketException? unsendableReply = null; + long admittedGeneration; lock (admitted.SessionGate) { - if (admitted.Session.ConnectRepeats(packet)) + admittedGeneration = admitted.Session.Generation; + if (!admitted.Session.ConnectRepeats(packet)) { + admitted.Session.ResetPeerState(); + } + else + { + repeats = true; CultNetRudpPacket reply; try { @@ -2851,10 +3003,15 @@ namespace GameCult.Networking _stats.PacketsDropped++; return true; } - SendPacket(admitted.RemoteEndPoint, reply); - return true; + unsendableReply = SendPacket(admitted.RemoteEndPoint, reply); } } + if (repeats) + { + if (unsendableReply != null) + EndUnsendablePeer(admitted, unsendableReply, admittedGeneration); + return true; + } } var peer = new CultNetRudpSocketServerPeer( @@ -2867,9 +3024,20 @@ namespace GameCult.Networking ResendDelayMs = _resendDelayMs, MaxPendingReliablePackets = _maxPendingReliablePackets })); - _peers[peerKey] = peer; + SocketException? unsendableAccept; lock (peer.SessionGate) - SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + unsendableAccept = SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + if (unsendableAccept != null) + { + // The peer cannot be answered, so no session starts, and the one it replaced has + // already ended. + _peers.Remove(peerKey); + _stats.PacketsDropped++; + } + else + { + _peers[peerKey] = peer; + } if (admitted != null) { admitted.DisconnectReason = ReplacedPeerReason; @@ -2886,6 +3054,8 @@ namespace GameCult.Networking CultNetRudpReceiveResult? outcome; CultNetRudpPacket? acknowledgement = null; + SocketException? unsendable = null; + long existingGeneration = 0; lock (existingPeer.SessionGate) { try @@ -2902,20 +3072,15 @@ namespace GameCult.Networking if (outcome == null) { - try - { - SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); - } - catch (SocketException) - { - // Best effort: the session ends whether or not the peer hears it. - } + // The session ends whether or not the peer hears the goodbye. + SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); } else { + existingGeneration = existingPeer.Session.Generation; if (outcome.Reply != null) - SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); - SendPackets(existingPeer, outcome.ReadyToSend); + unsendable = SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); + unsendable ??= SendPackets(existingPeer, outcome.ReadyToSend, out _); if (packet.PacketType == CultNetRudpPacketType.Data) acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } @@ -2952,8 +3117,10 @@ namespace GameCult.Networking } delivered = _deliveredFrames.Count > 0 ? _deliveredFrames.Dequeue() : null; - if (acknowledgement != null) - SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (acknowledgement != null && unsendable == null) + unsendable = SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (unsendable != null) + EndUnsendablePeer(existingPeer, unsendable, existingGeneration); return true; } @@ -2963,23 +3130,118 @@ namespace GameCult.Networking /// public void PollResends() { + RetireUnsendablePeers(); if (_socket.Poll(0, SelectMode.SelectRead)) return; foreach (var peer in _peers.Values.ToArray()) { + SocketException? unsendable; + long generation; lock (peer.SessionGate) - SendPackets(peer, peer.Session.DueResends(NowMs())); + { + var due = peer.Session.DueResends(NowMs()); + generation = peer.Session.Generation; + unsendable = SendPackets(peer, due, out _); + } + if (unsendable != null) + EndUnsendablePeer(peer, unsendable, generation); } } - private void SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets) + // Stops at the first packet that can never be sent as built and returns its error and + // index; a transient failure is a lost datagram and the loop goes on. + private SocketException? SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets, out int failedIndex) { + failedIndex = 0; for (var index = 0; index < packets.Count; index++) { - SendPacket(peer.RemoteEndPoint, packets[index]); + var unsendable = SendPacket(peer.RemoteEndPoint, packets[index]); + if (unsendable != null) + { + failedIndex = index; + return unsendable; + } if ((index + 1) % WireBurstPackets == 0 && index + 1 < packets.Count) Thread.Yield(); } + return null; + } + + // An admitted peer owes a packet that can never be sent as built. Inside a poll that ends the + // peer's session, never the poll: the peer is told, and PeerDisconnected carries a reason that + // names the error. + private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error, long generation) + { + EndUnsendableSession(peer, error, generation); + RetireUnsendablePeers(); + } + + // Ends the session and tells the peer, on whichever thread found the failure. The peer stays in + // _peers until the polling thread retires it. The failure belongs to the generation the thread + // read under the gate when it sent; if that generation has already ended (another thread + // ended it over its own failure, or a new Connect from the endpoint replaced it), or the + // thread read it only after the session had ended (a Pong or ack answered on an ended + // session the polling thread has not yet retired), there is nothing left to end, and a + // goodbye would reach the session that now owns the endpoint or repeat the first one. The + // goodbye is built and sent under the gate, so no Connect is admitted between the two. + private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error, long generation) + { + lock (peer.SessionGate) + { + if (peer.Session.Generation != generation || peer.Session.Ended) + return; + var goodbye = peer.Session.EndUnsendable(error); + peer.UnsendableReason = goodbye.Payload; + SendPacket(peer.RemoteEndPoint, goodbye); + } + _unsendablePeers.Enqueue(peer); + } + + // Polling thread only. Ends the session of a peer whose delivered frame the application could not + // decode or handle, with a goodbye carrying the application's reason. The frame is already + // acknowledged, so the session cannot be kept: a retransmit would never be handled. The peer + // is told, the drop is counted, and the peer's frames still queued are not delivered. A + // session that already ended (a new Connect replaced it, or a send failure ended it) owes no + // goodbye, since the endpoint may belong to a new session; the call then returns false. + internal bool EndPeerSession(CultNetRudpSocketServerPeer peer, byte[] reason) + { + if (peer == null) throw new ArgumentNullException(nameof(peer)); + if (reason == null) throw new ArgumentNullException(nameof(reason)); + _stats.PacketsDropped++; + lock (peer.SessionGate) + { + if (peer.Session.Ended) + return false; + SendPacket(peer.RemoteEndPoint, peer.Session.EndWith(reason)); + } + + var kept = _deliveredFrames.Where(frame => !ReferenceEquals(frame.Peer, peer)).ToArray(); + _deliveredFrames.Clear(); + foreach (var frame in kept) + _deliveredFrames.Enqueue(frame); + + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + return true; + _peers.Remove(key); + peer.DisconnectReason = reason; + PeerDisconnected?.Invoke(peer); + return true; + } + + // Polling thread only. A peer a new Connect already replaced was reported then, and its key now + // names the peer that replaced it. + private void RetireUnsendablePeers() + { + while (_unsendablePeers.TryDequeue(out var peer)) + { + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + continue; + _peers.Remove(key); + peer.DisconnectReason = peer.UnsendableReason; + PeerDisconnected?.Invoke(peer); + } } /// @@ -2994,13 +3256,55 @@ namespace GameCult.Networking _socket.Dispose(); } - private void SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) + // A transient send failure is that peer's lost datagram: counted, never thrown. A reliable + // packet stays pending and is resent; an unreliable one was allowed to be lost. A permanent + // failure (CultNetRudpSession.IsPermanentSendError) is returned: the datagram can never be + // sent as built, and the caller decides between refusing the send and ending the peer's + // session. Only an encode failure, the listener's own bug, throws. + private SocketException? SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) { var wire = CultNetRudpPacketCodec.Encode(packet); - var sent = _socket.SendTo(wire, remoteEndPoint); - _stats.BytesSent += sent; + BeforeSend?.Invoke(remoteEndPoint, packet); + try + { + if (UnsendableAfter.TryGetValue(remoteEndPoint, out var remaining)) + { + if (remaining > 0) + { + UnsendableAfter[remoteEndPoint] = remaining - 1; + } + else + { + UnsendableAfter.Remove(remoteEndPoint); + throw new SocketException((int)SocketError.MessageSize); + } + } + if (FailingSendPeers.Contains(remoteEndPoint)) + throw new SocketException((int)SocketError.HostUnreachable); + var sent = _socket.SendTo(wire, remoteEndPoint); + _stats.BytesSent += sent; + } + catch (SocketException error) when (CultNetRudpSession.IsPermanentSendError(error)) + { + return error; + } + catch (SocketException) + { + _stats.SendFailures++; + } + return null; } + // Test seams, standing in for paths a loopback peer cannot be made to have. Peers whose every + // datagram fails to send transiently: + internal HashSet FailingSendPeers { get; } = new HashSet(); + + // Peers whose next datagram after this many fails permanently, once. + internal Dictionary UnsendableAfter { get; } = new Dictionary(); + + // Runs before each datagram is sent, to order another thread's work between two sends. + internal Action? BeforeSend { get; set; } + private static string RemoteKey(EndPoint endpoint) { return endpoint.ToString() ?? string.Empty; diff --git a/src/GameCult.Networking/GameCult.Networking.csproj b/src/GameCult.Networking/GameCult.Networking.csproj index 1eef5653..08ef2ed6 100644 --- a/src/GameCult.Networking/GameCult.Networking.csproj +++ b/src/GameCult.Networking/GameCult.Networking.csproj @@ -15,6 +15,10 @@ + + + + diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs new file mode 100644 index 00000000..b8de674f --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -0,0 +1,293 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading.Tasks; +using GameCult.Logging; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A schema server's poll serves every peer whatever one peer sends. A message that cannot be + /// decoded or handled is that peer's failure: its session ends, the drop is counted and logged, + /// and the drain goes on to the other peers. + /// + public class CultNetRudpSchemaServerPeerInputTests + { + private const uint ConnectionId = 0x10203096; + private const string Canary = "canary-5f1c9e0b"; + + private sealed class Client + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private sealed class CapturingLogger : ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static async Task Connect(RudpCultNetSchemaServer server) + { + var client = new Client + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + client.Socket.SendTo(CultNetRudpPacketCodec.Encode(client.Session.CreateConnect(0)), server.LocalEndPoint); + await server.PollAvailableAsync(8); + client.Session.Receive(Drain(client.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + return client; + } + + private static void SendSchema(RudpCultNetSchemaServer server, Client client, byte[] payload) => + client.Socket.SendTo( + CultNetRudpPacketCodec.Encode(client.Session.Send("schema", payload, new CultNetRudpSendOptions { Reliable = true, Ordered = true })), + server.LocalEndPoint); + + private static byte[] Malformed() => new byte[] { 0xc1 }; + + // Well-formed MessagePack naming a schema version no runtime knows. The decoder's own error + // quotes that version. + private static byte[] UnknownSchemaVersion() => + MessagePackSerializer.Serialize( + new Dictionary { ["schemaVersion"] = "cultnet." + Canary + ".v0", ["messageId"] = Canary }, + CultNetSchemaMessageSerialization.Options); + + private sealed class Scene : IDisposable + { + public RudpCultNetSchemaServer Server = null!; + public Client Bad = null!; + public Client Good = null!; + public CapturingLogger Logger = new(); + public List<(string MessageId, EndPoint From)> Served = new(); + public List Ended = new(); + + public void Dispose() + { + Server.Dispose(); + Bad.Socket.Dispose(); + Good.Socket.Dispose(); + } + } + + // The bad peer's frame is read first, so the drain reaches the good peer only by getting past it. + private static async Task BadThenGood(byte[] badPayload) + { + var scene = new Scene + { + Server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }) + }; + scene.Server.Logger = scene.Logger; + scene.Server.OnCultNet((message, peer) => + scene.Served.Add((message.MessageId, peer.RemoteEndPoint))); + scene.Server.PeerDisconnected += peer => scene.Ended.Add(((RudpCultNetSchemaServerPeer)peer).RemoteEndPoint); + scene.Bad = await Connect(scene.Server); + scene.Good = await Connect(scene.Server); + + SendSchema(scene.Server, scene.Bad, badPayload); + SendSchema(scene.Server, scene.Good, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "good-request" })); + return scene; + } + + [Test] + public async Task AMalformedPayloadDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task AnUnknownSchemaVersionDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task PollOnceDoesNotThrowForAPeersUndecodableMessage() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => + { + for (var attempt = 0; attempt < 16; attempt++) + await scene.Server.PollOnceAsync(); + }); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task APeerWhoseMessageCannotBeDecodedHasItsSessionEnded() + { + using var scene = await BadThenGood(Malformed()); + var dropped = scene.Server.Stats.PacketsDropped; + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The frame was already acknowledged, so the session cannot be kept: the peer is told, + // the listener forgets it, and the drop is counted. + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); + Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); + Assert.That(scene.Server.Stats.PacketsDropped, Is.EqualTo(dropped + 1)); + var goodbye = Drain(scene.Bad.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(scene.Bad.Session.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(Drain(scene.Good.Socket).Select(packet => packet.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + } + + // The peer's malformed message and a well-formed one after it arrive in one datagram's + // delivery: the second is held for order, and both are released together. The session ends + // on the first, so the second belongs to no session and is never handled. + [Test] + public async Task ARefusedPeersFramesStillQueuedAreNotHandled() + { + var logger = new CapturingLogger(); + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }); + server.Logger = logger; + var served = new List(); + server.OnCultNet((message, _) => served.Add(message.MessageId)); + var bad = await Connect(server); + var options = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + var malformed = bad.Session.Send("schema", Malformed(), options); + var after = bad.Session.Send("schema", CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "after-the-refused-one" }), options); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(after), server.LocalEndPoint); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(malformed), server.LocalEndPoint); + + await server.PollAvailableAsync(16); + + Assert.That(served, Is.Empty); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + bad.Socket.Dispose(); + } + + private static async Task<(RudpCultNetSchemaServer Server, Client Client, CapturingLogger Logger)> ServerWhoseHandlerAnswers( + byte[] response, int? maxFragmentBytes, int? unsendableAfter = null) + { + var logger = new CapturingLogger(); + var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId, + MaxFragmentBytes = maxFragmentBytes + }); + server.Logger = logger; + server.OnCultNet((_, peer) => + { + if (unsendableAfter is { } count) + server.Transport.UnsendableAfter[peer.RemoteEndPoint] = count; + server.Transport.SendSchema(peer.TransportPeer, response); + }); + var client = await Connect(server); + SendSchema(server, client, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "ask" })); + return (server, client, logger); + } + + // A handler whose own response can never be sent (larger than a datagram, unfragmented) fails + // on the server's account, not the peer's. The session still ends, but its goodbye does not say + // the peer's packet was refused. + [Test] + public async Task AHandlerFaultEndsTheSessionWithAReasonThatDoesNotBlameThePeer() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[70_000], maxFragmentBytes: null); + using (server) + { + await server.PollAvailableAsync(16); + + var goodbye = Drain(client.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(Encoding.UTF8.GetString(goodbye.Payload!), Is.EqualTo("server could not handle a message")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("handler failed (SocketException)")); + Assert.That(logger.Errors[0], Does.Contain("that peer's session ended.")); + } + client.Socket.Dispose(); + } + + // A response that fails after its first fragment has already ended the session over the send, + // with a goodbye that names the send failure. The backstop ends nothing more and says so. + [Test] + public async Task AHandlerWhoseResponseEndedTheSessionIsLoggedAsAlreadyEnded() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[3000], maxFragmentBytes: 1000, unsendableAfter: 1); + using (server) + { + await server.PollAvailableAsync(16); + await server.PollAvailableAsync(16); + + var goodbyes = Drain(client.Socket).Where(packet => packet.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(goodbyes[0].Payload!), Does.StartWith("packet could not be sent: ")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("that peer's session had already ended.")); + } + client.Socket.Dispose(); + } + + [Test] + public async Task TheDropIsLoggedByErrorTypeWithoutEchoingWhatThePeerSent() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The decoder's own error message quotes the unknown version, which is the peer's input. + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + Assert.That(scene.Logger.Errors[0], Does.Contain(nameof(MessagePackSerializationException))); + Assert.That(scene.Logger.Errors[0], Does.Not.Contain(Canary)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs new file mode 100644 index 00000000..81d255f7 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -0,0 +1,646 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// One peer whose datagrams cannot be sent must not stop the listener serving the others. The + /// failure is injected at the listener's one send function through FailingSendPeers. + /// + public class CultNetRudpSendFailureTests + { + private const uint ConnectionId = 0x10203099; + + private sealed class Peer + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public CultNetRudpSocketServerPeer Server = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static Peer Connect(CultNetRudpSocketTransportServer server, EndPoint serverEndPoint) + { + var peer = new Peer + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + peer.Socket.SendTo(CultNetRudpPacketCodec.Encode(peer.Session.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Session.Receive(Drain(peer.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + peer.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(peer.EndPoint)); + return peer; + } + + private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null, uint? initialSequence = null) + { + var serverSocket = Bind(); + var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5, + MaxFragmentBytes = maxFragmentBytes, + InitialSequence = initialSequence + }); + var x = Connect(server, serverSocket.LocalEndPoint!); + var y = Connect(server, serverSocket.LocalEndPoint!); + return (server, serverSocket.LocalEndPoint!, x, y); + } + + private static CultNetRudpPacket ReliableFrame(Peer peer, string text) => + peer.Session.Send("schema", Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + + [Test] + public void AFailedResendToOnePeerDoesNotStopTheOthersResends() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + // Both peers' first datagram is lost, so both have a reliable resend due. + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + // Y's path recovers; X's stays dead. X is visited first. + server.FailingSendPeers.Remove(y.EndPoint); + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(2), "a send failure ends no session"); + } + } + + [Test] + public void AFailedAckToOnePeerDoesNotStopTheListenerServingTheOthers() + { + var (server, listenerEndPoint, x, y) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + y.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(y, "from-y")), listenerEndPoint); + + var frames = new List(); + for (var attempt = 0; attempt < 20 && frames.Count < 2; attempt++) + { + Assert.DoesNotThrow(() => + { + if (server.ReceiveOnce() is { } frame) + frames.Add(Encoding.UTF8.GetString(frame.Frame.Payload)); + }); + } + + Assert.That(frames, Is.EquivalentTo(new[] { "from-x", "from-y" })); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(1), "X's ack was counted as lost"); + Assert.That(Drain(y.Socket).Select(p => p.PacketType), Does.Contain(CultNetRudpPacketType.Ack)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + } + } + + // A permanent send failure is one that can never succeed for the datagram as built. A + // caller-directed send throws it and queues nothing; inside a poll it ends that peer's + // session and never the poll. + + private static bool IsUnsendableReason(byte[]? reason) + { + var text = Encoding.UTF8.GetString(reason ?? Array.Empty()); + return text.StartsWith("packet could not be sent: ", StringComparison.Ordinal) + && text.Length > "packet could not be sent: ".Length; + } + + private static List RecordDisconnects(CultNetRudpSocketTransportServer server) + { + var ended = new List(); + server.PeerDisconnected += ended.Add; + return ended; + } + + [Test] + public void AnOversizedSendThrowsTheErrorAndQueuesNothing() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + // Each peer's own Accept is still awaiting its ack; the refused send adds nothing to it. + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + var otherOutstanding = y.Server.Session.OutstandingReliablePacketCount; + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[70_000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString()); + Assert.That(server.Stats.SendFailures, Is.Zero); + Assert.That(server.Stats.FramesSent, Is.Zero); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session"); + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing is pending or queued to resend"); + + // The refused send consumed no sequence: the next frame is delivered in order instead + // of waiting behind a gap that never fills. + server.SendSchema(x.Server, "next"); + var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data); + var delivered = x.Session.Receive(data, 1).Delivered; + Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" })); + Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(otherOutstanding)); + } + } + + [Test] + public void ARefusedSendWithdrawsTheFragmentsTheWindowQueuedAsWell() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1); + using (server) + { + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + server.UnsendableAfter[x.EndPoint] = 0; + + // 1,100 one-byte fragments: the flow window admits the first ones and queues the rest, + // and the first datagram can never be sent. + Assert.Throws(() => server.SendSchema(x.Server, new byte[1_100])); + + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing of the refused send stays queued"); + } + } + + [Test] + public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 1; + + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[3000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True); + // The send ends the session and tells the peer; it leaves the peer for the polling + // thread to retire, because only that thread changes the peer set. + Assert.That(x.Server.Connected, Is.False); + Assert.That(Drain(x.Socket).Last().PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(ended, Is.Empty); + + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + + [Test] + public void APeerEndedByASendOnAnotherThreadIsRetiredByThePollingThread() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var retiredOn = new List(); + server.PeerDisconnected += _ => retiredOn.Add(Environment.CurrentManagedThreadId); + server.UnsendableAfter[x.EndPoint] = 1; + + SocketException? thrown = null; + var sender = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException error) { thrown = error; } + }); + sender.Start(); + sender.Join(); + Assert.That(thrown, Is.Not.Null); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the sending thread leaves the peer set alone"); + + server.PollResends(); + Assert.That(retiredOn, Is.EqualTo(new[] { Environment.CurrentManagedThreadId })); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + + // A client that connects again from the same endpoint replaces the peer the listener held. A + // caller still holding the replaced peer (a handler still running for it, say) reaches nothing + // of the session that replaced it: no data and no goodbye. + + /// X connects again as a new session, and the handshake completes. + private static (CultNetRudpSocketServerPeer Stale, CultNetRudpSocketServerPeer Replacing) Reconnect( + CultNetRudpSocketTransportServer server, EndPoint listenerEndPoint, Peer x) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + server.ReceiveOnce(); + x.Session.Receive(Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + x.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(x.Server, Is.Not.SameAs(stale)); + Assert.That(x.Session.Connected, Is.True); + return (stale, x.Server); + } + + [Test] + public void ASendOnAReplacedPeerThrowsAndNoGoodbyeReachesTheSessionThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + var ended = RecordDisconnects(server); + Assert.That(stale.Connected, Is.False, "the replaced peer's session ended with the replacement"); + + // Had the stale send gone out, a failure after its first fragment would end the + // session that sent it and say goodbye to the endpoint. + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + Assert.That(Drain(x.Socket), Is.Empty, "nothing of the stale send reached the client"); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + Assert.That(ended, Is.Empty); + } + } + + // With a fixed initial sequence the stale frame would take the sequence the real one gets, and + // the real one would be dropped as a duplicate. With a drawn one it would land far ahead of + // the new session's window, and nothing after it would be delivered. + [TestCase(1000u)] + [TestCase(null)] + public void ASendOnAReplacedPeerDeliversNothingAndTheReplacingPeersFrameArrives(uint? initialSequence) + { + var (server, listenerEndPoint, x, _) = TwoPeers(initialSequence: initialSequence); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + + Assert.Throws(() => server.SendSchema(stale, "STALE")); + server.SendSchema(replacing, "REAL"); + + var delivered = Drain(x.Socket) + .SelectMany(packet => x.Session.Receive(packet, 1).Delivered) + .Select(frame => Encoding.UTF8.GetString(frame.Payload)) + .ToList(); + Assert.That(delivered, Is.EqualTo(new[] { "REAL" })); + } + } + + // A send that fails after its first fragment ends the generation it began in, and only that + // one. Here X connects again while the send is under way, and the polling thread admits the + // new Connect before the fragment that can never be sent. The hook runs that poll inside the + // send so the order is fixed; in production it runs on the polling thread between the send + // and the failure's handling. The goodbye belonged to the ended generation, and the client + // must not hear it: it would believe itself disconnected while the listener holds it + // connected. + [Test] + public void ASendFailureAfterANewConnectReplacedThePeerSaysNoGoodbyeToTheNewSession() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + var fragments = 0; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Data || ++fragments < 2) + return; + server.BeforeSend = null; + server.ReceiveOnce(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + var packets = Drain(x.Socket); + Assert.That(packets.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect), + "the ended generation's goodbye reached the session that replaced it"); + x.Session.Receive(packets.Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + Assert.That(x.Session.Connected, Is.True); + var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(replacing, Is.Not.SameAs(stale)); + Assert.That(replacing.Connected, Is.True); + } + } + + // A caller's send ends X's session over a fragment it could never send. Before the listener + // retires X, the polling thread answers X's Ping on the ended session, and that Pong can never + // be sent either. The polling thread read the generation only after the session ended, so the + // generation alone cannot tell it the session is gone. It says no second goodbye, and the + // first failure's reason stands. The hook admits the Ping while the first goodbye is on its way, + // after the session ended and before the peer is queued for retirement. + [Test] + public void AFailureAnsweringAPingOnAnAlreadyEndedSessionSaysNoSecondGoodbye() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + byte[]? firstReason = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + firstReason = x.Server.UnsendableReason; + server.UnsendableAfter[x.EndPoint] = 0; + server.ReceiveOnce(); + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreatePing(Encoding.UTF8.GetBytes("ping"))), listenerEndPoint); + server.UnsendableAfter[x.EndPoint] = 1; + + Assert.Throws(() => server.SendSchema(x.Server, new byte[3000])); + server.ReceiveOnce(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(firstReason, Is.Not.Null); + Assert.That(x.Server.UnsendableReason, Is.SameAs(firstReason), "the second failure overwrote the first's reason"); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(x.Server.DisconnectReason, Is.SameAs(firstReason)); + } + } + + // The goodbye for a failed send is sent under the session's gate. The polling thread, admitting + // a new Connect from X, waits on that gate, so the ended generation's goodbye can never follow + // the new session's Accept. The hook starts the polling thread just before the goodbye leaves + // and gives it a second to get ahead if nothing holds it back. + [Test] + public void AFailedSendsGoodbyeLeavesBeforeANewConnectFromTheEndpointIsAdmitted() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + Thread? poller = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + poller = new Thread(() => server.ReceiveOnce()); + poller.Start(); + poller.Join(1000); + }; + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + Assert.That(poller, Is.Not.Null); + poller!.Join(); + + var types = Drain(x.Socket).Select(p => p.PacketType).ToList(); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Disconnect)); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Accept)); + Assert.That(types.IndexOf(CultNetRudpPacketType.Disconnect), Is.LessThan(types.IndexOf(CultNetRudpPacketType.Accept)), + "the ended generation's goodbye followed the new session's Accept"); + } + } + + // An application ending a peer's session (a message it could not decode or handle) says no + // goodbye to a session that already ended: a new Connect replaced it, and the endpoint now + // belongs to the session that replaced it. + [Test] + public void EndingAReplacedPeersSessionSaysNoGoodbyeToTheSessionThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + var ended = RecordDisconnects(server); + + Assert.That(server.EndPeerSession(stale, Encoding.UTF8.GetBytes("application reason")), Is.False); + + Assert.That(Drain(x.Socket).Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(ended, Is.Empty); + } + } + + [Test] + public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + // The polling thread acknowledges X's frame outside the session's gate. Just before + // that ack goes out, a caller's send to X fails after its first fragment and ends the + // session; then the ack can never be sent either. + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Ack) + return; + server.BeforeSend = null; + server.UnsendableAfter[x.EndPoint] = 1; + var caller = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException) { } + }); + caller.Start(); + caller.Join(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + server.ReceiveOnce(); + server.PollResends(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + + [Test] + public void APermanentFailureInResendsEndsOnlyThatPeersSession() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + + // X's datagram can now never be sent as built; Y's path recovers. X is visited first. + server.FailingSendPeers.Clear(); + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + Assert.That(server.Peers, Is.EqualTo(new[] { y.Server })); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + var goodbye = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(IsUnsendableReason(goodbye.Payload), Is.True); + } + } + + [Test] + public void APermanentFailureAcknowledgingAFrameEndsTheSessionAfterTheFrameIsDelivered() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 0; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + + CultNetRudpSocketServerFrame? frame = null; + Assert.DoesNotThrow(() => frame = server.ReceiveOnce()); + + Assert.That(frame, Is.Not.Null, "the frame is delivered before the session ends"); + Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("from-x")); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void APermanentFailureAnsweringARepeatedConnectEndsTheSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var ended = RecordDisconnects(server); + var peer = Bind(); + var connect = CultNetRudpPacketCodec.Encode( + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)); + peer.SendTo(connect, serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + var admitted = server.Peers.Single(); + + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo(connect, serverSocket.LocalEndPoint!); + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(ended, Is.EqualTo(new[] { admitted })); + Assert.That(IsUnsendableReason(admitted.DisconnectReason), Is.True); + } + + [Test] + public void AConnectWhoseAcceptCanNeverBeSentStartsNoSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var peer = Bind(); + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo( + CultNetRudpPacketCodec.Encode(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)), + serverSocket.LocalEndPoint!); + + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + } + + [Test] + public void APeerEndedInTheResendLoopIsSentNothingMoreFromIt() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + server.SendSchema(x.Server, "one"); + server.SendSchema(x.Server, "two"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + server.PollResends(); + + // Both resends are due. The first can never be sent and ends the session; the + // goodbye is the only other datagram attempted at X. + Assert.That(server.Stats.SendFailures, Is.EqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + + [Test] + public void AHealthySendCountsItsBytesAndNoFailure() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + var before = server.Stats.BytesSent; + server.SendSchema(x.Server, "hello"); + var received = Drain(x.Socket).Sum(p => CultNetRudpPacketCodec.Encode(p).Length); + Assert.That(received, Is.GreaterThan("hello".Length)); + Assert.That(server.Stats.BytesSent - before, Is.EqualTo(received)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void OnlyADatagramThatCanNeverBeSentIsAPermanentFailure() + { + foreach (var permanent in new[] { SocketError.MessageSize, SocketError.InvalidArgument, SocketError.AddressFamilyNotSupported }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)permanent)), Is.True, permanent.ToString()); + foreach (var transient in new[] + { + SocketError.HostUnreachable, SocketError.NetworkUnreachable, SocketError.NoBufferSpaceAvailable, + SocketError.WouldBlock, SocketError.ConnectionReset, SocketError.ConnectionRefused, + SocketError.AccessDenied, SocketError.NetworkDown, SocketError.AddressNotAvailable, + SocketError.Interrupted, SocketError.TimedOut, SocketError.Shutdown + }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)transient)), Is.False, transient.ToString()); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs index 9b6959c0..e4c31d64 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs @@ -796,8 +796,11 @@ namespace GameCult.Networking.Tests serverThread.Join(); } - Assert.That(logger.Errors, Has.Some.Contains("dispatch-backstop-probe"), - "the backstop must log the handler's own exception message, not swallow it silently"); + // The log names the fault's type. The fault's own message can quote what the peer sent, + // so it is never logged. + Assert.That(logger.Errors, Has.Some.Contains(nameof(InvalidOperationException)), + "the backstop must log the handler's fault, not swallow it silently"); + Assert.That(logger.Errors, Has.None.Contains("dispatch-backstop-probe")); } private sealed class CapturingLogger : ILogger diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 8bf59746..5a1f570f 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -1141,6 +1141,196 @@ namespace GameCult.Networking.Tests Assert.That(delivered[0].Payload, Is.EqualTo(payload)); } + // The sender's flow window: a reliable packet goes on the wire only while its sequence is at most 1,023 + // above the lowest unacked one and the payload above that sequence stays within 4 MiB. + private const int Mib = 1024 * 1024; + + private static CultNetRudpSession FlowSession(uint initialSequence) => + new CultNetRudpSession(new CultNetRudpSessionOptions + { + ConnectionId = 0x464c4f57, + InitialSequence = initialSequence, + ResendDelayMs = 25 + }); + + // A sender and a receiver that have shaken hands: the receiver's watermark is seeded by the sender's + // Connect, and the sender's Connect is acknowledged, so nothing is pending on the sender. + private static (CultNetRudpSession Sender, CultNetRudpSession Receiver) ConnectedFlowPair(uint senderInitialSequence, uint receiverInitialSequence) + { + var sender = FlowSession(senderInitialSequence); + var receiver = FlowSession(receiverInitialSequence); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); + return (sender, receiver); + } + + private static CultNetRudpSession ConnectedFlowSession(uint initialSequence) => ConnectedFlowPair(initialSequence, 900).Sender; + + private static IReadOnlyList FlowSend(CultNetRudpSession session, int payloadBytes) => + session.SendMany("state", new byte[payloadBytes], new CultNetRudpSendOptions { Reliable = true }); + + private static CultNetRudpPacket FlowAck(uint sequence) => + new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Ack, ConnectionId = 0x464c4f57, Ack = sequence, ChannelId = "control" }; + + private static CultNetRudpPacket LoseGAndFillTheSpan(CultNetRudpSession session) + { + var g = FlowSend(session, 1).Single(); + for (uint offset = 1; offset <= 1023; offset++) + { + Assert.That(FlowSend(session, 1).Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + offset })); + session.Receive(FlowAck(g.Sequence + offset), 1); + } + return g; + } + + [Test] + public void RudpSession_ALostPacketHoldsTheSender1023SequencesAheadAndIsStillDelivered() + { + var (sender, receiver) = ConnectedFlowPair(1, 100); + var g = FlowSend(sender, 1).Single(); + var admittedAfterG = 0; + for (var index = 0; index < 4200; index++) + { + foreach (var packet in FlowSend(sender, 1)) + { + admittedAfterG++; + receiver.Receive(packet, 1); + sender.Receive(receiver.CreateAck(), 1); + } + } + Assert.That(admittedAfterG, Is.EqualTo(1023)); + Assert.That(sender.QueuedReliablePacketCount, Is.EqualTo(4200 - 1023)); + + var retransmit = sender.DueResends(1000).Single(packet => packet.Sequence == g.Sequence); + var delivered = receiver.Receive(retransmit, 1000).Delivered; + Assert.That(delivered.Select(frame => frame.Sequence), Is.EqualTo(new[] { g.Sequence })); + + var promoted = sender.Receive(receiver.CreateAckForReceived(g.Sequence), 1001).ReadyToSend; + Assert.That(promoted.First().Sequence, Is.EqualTo(g.Sequence + 1024)); + } + + [Test] + public void RudpSession_ADirectSendPastTheSpanIsRefusedWithoutConsumingASequence() + { + var sender = ConnectedFlowSession(1); + var g = LoseGAndFillTheSpan(sender); + Assert.Throws(() => sender.Send("state", new byte[1], new CultNetRudpSendOptions { Reliable = true })); + Assert.That(sender.QueuedReliablePacketCount, Is.Zero); + sender.Receive(FlowAck(g.Sequence), 2); + var next = sender.Send("state", new byte[1], new CultNetRudpSendOptions { Reliable = true }); + Assert.That(next.Sequence, Is.EqualTo(g.Sequence + 1024)); + } + + [Test] + public void RudpSession_PayloadAboveTheLowestUnackedSequenceIsBoundedBy4MiB() + { + var sender = ConnectedFlowSession(1); + var g = FlowSend(sender, Mib).Single(); + // The lowest unacked packet's own bytes do not count: four more MiB fit above it. + for (var index = 0; index < 4; index++) + Assert.That(FlowSend(sender, Mib), Has.Count.EqualTo(1)); + Assert.That(FlowSend(sender, Mib), Is.Empty); + Assert.That(FlowSend(sender, Mib), Is.Empty); + Assert.That(sender.QueuedReliablePacketCount, Is.EqualTo(2)); + + // Acking g moves the floor up one packet: exactly one MiB more fits. + var promoted = sender.Receive(FlowAck(g.Sequence), 1).ReadyToSend; + Assert.That(promoted.Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 5 })); + Assert.That(sender.QueuedReliablePacketCount, Is.EqualTo(1)); + } + + [Test] + public void RudpSession_APacketLargerThanTheWindowGoesOutAloneAndWaitsBehindAnythingUnacked() + { + Assert.That(FlowSend(ConnectedFlowSession(1), 5 * Mib), Has.Count.EqualTo(1)); + var sender = ConnectedFlowSession(1); + var g = FlowSend(sender, 1).Single(); + Assert.That(FlowSend(sender, 5 * Mib), Is.Empty); + var promoted = sender.Receive(FlowAck(g.Sequence), 1).ReadyToSend; + Assert.That(promoted.Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 1 })); + } + + [Test] + public void RudpSession_ASmallPacketDoesNotOvertakeAQueuedLargeOne() + { + var sender = ConnectedFlowSession(1); + var g = FlowSend(sender, 1).Single(); + Assert.That(FlowSend(sender, 3 * Mib), Has.Count.EqualTo(1)); + Assert.That(FlowSend(sender, 2 * Mib), Is.Empty); + // A direct send refuses instead of queueing behind it, and takes no sequence. + Assert.Throws(() => sender.Send("state", new byte[1], new CultNetRudpSendOptions { Reliable = true })); + Assert.That(sender.QueuedReliablePacketCount, Is.EqualTo(1)); + // One byte would fit above g, but the 2 MiB packet is ahead of it. + Assert.That(FlowSend(sender, 1), Is.Empty); + var promoted = sender.Receive(FlowAck(g.Sequence), 1).ReadyToSend; + Assert.That(promoted.Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 2, g.Sequence + 3 })); + } + + [Test] + public void RudpSession_AcknowledgedBytesAboveALostPacketStillCountAgainst4MiB() + { + var sender = ConnectedFlowSession(1); + var g = FlowSend(sender, 1).Single(); + // g is lost. Each 1 MiB frame above it is delivered and acknowledged, and the receiver still holds it + // behind the gap, so it keeps counting. + for (uint offset = 1; offset <= 4; offset++) + { + Assert.That(FlowSend(sender, Mib).Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + offset })); + sender.Receive(FlowAck(g.Sequence + offset), 1); + } + Assert.That(FlowSend(sender, Mib), Is.Empty); + Assert.That(sender.QueuedReliablePacketCount, Is.EqualTo(1)); + Assert.Throws(() => sender.Send("state", new byte[Mib], new CultNetRudpSendOptions { Reliable = true })); + + // g arrives: nothing is held behind a gap any more, and the queue drains. + var promoted = sender.Receive(FlowAck(g.Sequence), 2).ReadyToSend; + Assert.That(promoted.Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 5 })); + // The bytes above the old gap no longer count: another MiB fits. + Assert.That(FlowSend(sender, Mib).Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 6 })); + } + + [Test] + public void RudpSession_AcknowledgedBytesCountAgainstTheLowestUnackedSequenceAsItAdvances() + { + var sender = ConnectedFlowSession(1); + var g = FlowSend(sender, 1).Single(); + var h = FlowSend(sender, 1).Single(); + for (var index = 0; index < 3; index++) + FlowSend(sender, Mib); + // g and the 3 MiB above h are acknowledged; h is lost. + foreach (uint offset in new uint[] { 2, 3, 4, 0 }) + sender.Receive(FlowAck(g.Sequence + offset), 1); + Assert.That(FlowSend(sender, Mib).Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 5 })); + // 4 MiB sit above h, acknowledged or not: one more byte does not fit. + Assert.That(FlowSend(sender, 1), Is.Empty); + // h arrives, the floor moves to g + 5, and the byte fits. + var promoted = sender.Receive(FlowAck(h.Sequence), 2).ReadyToSend; + Assert.That(promoted.Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + 6 })); + } + + // A server that accepts a fresh Connect may send before its Accept is acknowledged, so no acknowledgement of + // the new generation has moved the floor yet. Bytes the ended generation had acknowledged above its own lost + // packet belong to no receiver's hold any more and must not count. + [Test] + public void RudpSession_ANewGenerationOnTheServerDoesNotInheritAcknowledgedBytes() + { + var server = FlowSession(500); + var accept = server.AcceptConnect(FlowSession(1).CreateConnect(0), 0); + server.Receive(FlowAck(accept.Sequence), 0); + var g = FlowSend(server, 1).Single(); + for (uint offset = 1; offset <= 4; offset++) + { + Assert.That(FlowSend(server, Mib).Select(packet => packet.Sequence), Is.EqualTo(new[] { g.Sequence + offset })); + server.Receive(FlowAck(g.Sequence + offset), 1); + } + Assert.That(FlowSend(server, Mib), Is.Empty); + + var nextAccept = server.AcceptConnect(FlowSession(200_000).CreateConnect(2), 2); + Assert.That(nextAccept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + // The Accept is unacknowledged and the only thing pending: 1 MiB fits above it. + var sent = server.Send("state", new byte[Mib], new CultNetRudpSendOptions { Reliable = true }); + Assert.That(sent.Sequence, Is.EqualTo(nextAccept.Sequence + 1)); + } + [Test] public void RudpSocketTransport_HandshakesAndCarriesReliableOrderedSchemaFrames() { @@ -5262,10 +5452,49 @@ namespace GameCult.Networking.Tests #pragma warning restore CS0618 } - [Test] - public void CultMeshRudpClient_DefaultBindAllowsRemoteRoutes() + private static string? RudpClientBoundHost(string endpoint, string? bindHost = null) { - Assert.That(new CultMeshRudpSocketOptions().BindHost, Is.EqualTo("0.0.0.0")); + using var client = CultMesh.CreateRudpClient( + "csharp-cultmesh-bind-client", + 0x10203060u, + endpoint, + new CultMeshRudpSocketOptions { BindHost = bindHost }); + return client.Profile.Transports[0].Host; + } + + // A client bound to loopback cannot send to another host on Windows, so a client for a + // remote endpoint binds the unspecified address of its family. + [Test] + public void CultMeshRudpClient_ForARemoteEndpoint_BindsTheUnspecifiedAddress() + { + Assert.That(RudpClientBoundHost("rudp://10.77.0.1:17872"), Is.EqualTo("0.0.0.0")); + Assert.That(RudpClientBoundHost("rudp://[2001:db8::1]:17872"), Is.EqualTo("::")); + } + + [Test] + public void CultMeshRudpClient_ForALoopbackEndpoint_BindsLoopback() + { + Assert.That(RudpClientBoundHost("rudp://127.0.0.1:17872"), Is.EqualTo("127.0.0.1")); + Assert.That(RudpClientBoundHost("rudp://[::1]:17872"), Is.EqualTo("::1")); + } + + [Test] + public void CultMeshRudpClient_BindsAnExplicitBindHostWhateverTheEndpoint() + { + Assert.That(RudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1"), Is.EqualTo("127.0.0.1")); + Assert.That(RudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0"), Is.EqualTo("0.0.0.0")); + } + + [Test] + public void CultMeshRudpServer_BindsEveryIPv4InterfaceUnlessToldOtherwise() + { + using var anywhere = CultMesh.CreateRudpServer("csharp-cultmesh-bind-server", 0x10203061u); + using var loopback = CultMesh.CreateRudpServer( + "csharp-cultmesh-bind-server", + 0x10203062u, + new CultMeshRudpSocketOptions { BindHost = "127.0.0.1" }); + Assert.That(anywhere.Profile.Transports[0].Host, Is.EqualTo("0.0.0.0")); + Assert.That(loopback.Profile.Transports[0].Host, Is.EqualTo("127.0.0.1")); } [Test]