diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 8d2172c0..66bb450e 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -104,6 +104,16 @@ Not implemented yet: - cross-runtime rollback/resimulation helpers for simulation frames - declared CRDT merge policies +## Error Codes + +`cultnet.error.v0` carries an optional machine-readable `code`. The set of codes is a closed enum +(`cultnet.error.schema.json`): Rust and TypeScript refuse a message whose code they do not know, they do +not guess. Adding a code is therefore a coordinated change across every runtime, made together: the schema (both +copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in +`packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are +`selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported` and +`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns). + ## Live Invariants - CultCache owns document schema, identity, local indexes, and local diffing. diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index 15880fa1..59764338 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -26,9 +26,9 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/docs/cultcache-publication-cut.md b/docs/cultcache-publication-cut.md index ed281978..8a7cb2fc 100644 --- a/docs/cultcache-publication-cut.md +++ b/docs/cultcache-publication-cut.md @@ -826,3 +826,15 @@ Recorded, not forked: - **Crash between the store write and the log append** leaves the log lagging the store, and nothing detects it; a restarted primary reuses that sequence (A2's divergence reached by a plain kill). The spec called this lag detectable; for a crash it is not. +- **Cut 4 final Soul (4d), recorded 2026-09-30 (Self), not fixed in Cut 4:** + - **F2, read cost.** `CultNetDatabase.Serves` (`:1980-1989`) runs per row in `GetAll`/`GetByName`/`GetByIndex` for every + layout except a single owns-everything shard: a reflection `GetTrackedKey`, a gate lock and a shard re-scan per row. + `database.GetAll` measured 24.8x `cache.GetAll` on 20,000 rows under a key-prefixed shard. Fix: filter by key inside + the cache enumeration (the `rowFilter` path already has keys) and hoist the owns-everything check. + - **F4, dispose drain edges.** An observer that removed its stash entry before the drain can publish after `Dispose` + returns (dropped); a throwing `_cacheJournal.Dispose()` skips the drain. Neither double-publishes or deadlocks. + - **F5, refusal naming.** A write whose schema is owned but whose key is outside the shard's prefix also answers + `unowned_schema` with details `{field:"schemaId"}`; the peer cannot tell a key refusal from a schema refusal. + - **Pre-existing, separate task.** Python emits error code `unsupported_schema_version` + (`cultnet-py/.../interop_peer.py:415`, `cultmesh-py/.../server.py:161`), which is not in the error-code enum; TS and + Rust refuse the whole message. diff --git a/docs/cultnet-error-contract-cut.md b/docs/cultnet-error-contract-cut.md new file mode 100644 index 00000000..923cf51a --- /dev/null +++ b/docs/cultnet-error-contract-cut.md @@ -0,0 +1,644 @@ +# CultNet error contract: cut map + +Status: cut map, not started. Imagination pass 1, 2026-09-30, against CultLib `main` at `8fc74c70`. +This document owns the means. There is no separate target doc: section 1 holds the ends until the +operator rules on section 3. After that, Self moves the rulings and invariants into +`contracts/cultnet/cultnet-distributed-database.md` ("Error Codes") and the schema's own description. + +Open: operator questions Q1-Q8 (section 3). No cut is dispatched until Q1, Q2, Q3 and Q8 are ruled. Q4-Q7 +change only single emit sites and can be ruled during Cut 1. + +Depends on: `hands/cultmesh-put-serve-bound` merging first. It introduces `document_unservable` in Python +(`fd394d54`), a codeless unservable refusal in cultmesh-ts (`0eb7434a`) and `CultMeshRudpRejectionReason` in +cultmesh-rs (`c9cdad12`, `85288a5a`). The cuts below convert those sites. Their anchors are by commit, not by +line, because the branch is not on main. + +Map branch: `main`. + +--- + +## 0. What the Body does today (probed) + +Every claim in this table was established by running code. The C#, Rust and TypeScript probes ran on +Yggdrasil through `ygg-verify.sh` at `8fc74c70`. The Python bytes came from `msgpack.packb` on the exact +dicts `cultmesh_py/server.py:573-589` and `cultmesh-ts/src/index.ts:5921-5926` build. The six probe inputs: + +| Input | Bytes from | +|---|---| +| `cs_cursor_stale` | Rust `tests/error_message.rs` captured C# hex (`cursor_stale`, details `{asOf:1,current:2}`) | +| `cs_routing` | C# `new CultNetErrorMessage { Error, RoutingHint = { Reason="not_primary", Shard=... } }` serialized with `CultNetSchemaMessageSerialization` | +| `cs_codeless` | C# `new CultNetErrorMessage { Error = "boom" }` | +| `cs_unknown_code` | C# with `Code = "document_unservable"`, empty `CultNetErrorDetails` | +| `py_snapshot_byte_limit` | the Python `_error_response` shape: `messageId`, `code: "snapshot_byte_limit_exceeded"`, `details {responseBytes, maxSnapshotBytes}` | +| `ts_two_key` | the cultmesh-ts shape `{schemaVersion, error}` | + +Decoded by: + +| Input | C# `CultNetSchemaMessageSerialization.Deserialize` | Rust `decode_cultnet_message_from_slice` | TS `parseCultNetMessage` | +|---|---|---|---| +| `cs_cursor_stale` | ok | ok, typed | ok | +| `cs_routing` | ok | ok, **routingHint dropped** (`code: None, details: None`) | ok (untyped passthrough) | +| `cs_codeless` | ok | ok | ok | +| `cs_unknown_code` | ok (free string) | **Err: "not a recognized cultnet.error.v0 code"** | **throws: /code must be one of the allowed values** | +| `py_snapshot_byte_limit` | ok, but **details silently become an all-null object** (`responseBytes`, `maxSnapshotBytes` lost), `messageId` dropped | **Err: not a recognized code** | **throws** (missing routingHint, extra messageId, code enum, details keys) | +| `ts_two_key` | ok | ok | **throws: missing routingHint, code, details** — TypeScript cannot parse the errors TypeScript sends | + +Captured C# bytes for the two shapes Rust's vector file does not hold (for Hands' reuse): + +- `cs_codeless`: `85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72A4626F6F6DAB726F7574696E6748696E74C0A4636F6465C0A764657461696C73C0` +- `cs_routing`: see the probe; the shard descriptor writes ten keys, nil included. + +Other findings, from reading source at `8fc74c70`: + +1. **The interop CI lane has been red for 40 consecutive `main` runs.** It fails at "Verify CultNet schema + contracts stayed consolidated" (`.github/workflows/cultnet-interop.yml:67-90`). That step forbids every + `*.schema.json` outside `contracts/cultnet/`, and R-AB deliberately vendors 22 copies into + `packages/cultnet-rs/contracts/cultnet/`. Those copies are held byte-equal by + `packages/cultnet-rs/tests/schema_discovery_contracts.rs:74-99`. Nothing after that step (every + cross-runtime interop test) has run on main in that time. Run `36714768445` is the latest. +2. **The schema contradicts itself.** `contracts/cultnet/cultnet.error.schema.json:29` says the code is + "Additive: a v0 peer that does not know this field ignores it" and, in the same sentence, "a closed + enum: Rust and TypeScript refuse". The closed-enum text came from Soul pass 4d (`3f942a57`). It + described what the code did; no operator ruled it. `docs/` holds no ruling on unknown codes. +3. **Emitters that break the schema:** + - cultmesh-ts sends `{schemaVersion, error}` only, at `packages/cultmesh-ts/src/index.ts:5875-5878`, + `5885-5888`, `5913-5916` and `5922-5925`, and on the branch at `0eb7434a`. + - cultmesh-py `_error_response` (`packages/cultmesh-py/src/cultmesh_py/server.py:573-589`) adds a + top-level `messageId`, omits `routingHint`, omits `code` and `details` rather than sending nil, and + sends eight out-of-enum codes: `unsupported_schema_version` (`:161`), `malformed_document_put` + (`:358`), `unregistered_document_put` (`:365`), `malformed_document_delete` (`:386`), + `unregistered_document_delete` (`:393`), `snapshot_document_limit_exceeded` (`:558`), + `snapshot_byte_limit_exceeded` (`:567`) and `simulation_observations_disabled` (`:596`). The branch + adds a ninth, `document_unservable` (`fd394d54`). + - `packages/cultnet-py/src/cultnet_py/interop_peer.py:409-417` also sends `unsupported_schema_version`. + R-2 (`docs/cultnet-selection-cut.md:2046-2053`) required that code, but it never entered the schema + enum. +4. **A second schema authority.** `packages/cultnet-py/src/cultnet_py/schema_catalog.py:17`, `:272` and + `:357-362` publish a hand-built `cultnet.error.v0` descriptor under the canonical `$id`. It has a + different shape: `messageId`, `code` as any string, `details` open, and only `schemaVersion` and `error` + required. The same file does this for every wire message (about 350 lines). This is recorded as + follow-up F-1, not cut here. +5. **Silent refusals.** cultmesh-rs's RUDP document server returns every rejection to the local caller only + (`packages/cultmesh-rs/src/rudp_document_server.rs:470-590`, `CultMeshRudpApplicationRejection`). The + peer gets nothing and times out. On the branch this includes the new `DocumentUnservable`. +6. **Receivers drop the code.** + - Every C# receiver turns an error into `new IOException(error.Error)` or + `new InvalidOperationException(error.Error)`, so a caller cannot branch on the code. The sites: + `src/GameCult.Mesh/CultMeshBodySessions.cs:310`, `CultMeshContentSessions.cs:178/256-260`, + `CultMeshPeerExchange.cs:430/466`, `CultMeshSnapshots.cs:91/107/222-226/743` and + `CultMeshVerseDiscovery.cs:273`, plus `src/GameCult.Networking/CultNetShardReplication.cs:440/574`. + - Rust reads `Error { error, .. }` at `packages/cultmesh-rs/src/lib.rs:823`. + - TS types `CultNetErrorMessage` as `{schemaVersion, error}` only (`packages/cultnet-ts/src/contracts.ts:167-170`). + - Kotlin has no error type. A request that gets an error back fails a + `require(response.schemaVersion == ...)` (`packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt:1278`, + `:1334`), and the peer's text is lost. +7. **No correlation.** Only Python echoes `messageId`. + - `cultmesh-browser` rejects an error only when exactly one operation is pending + (`packages/cultmesh-browser/src/index.ts:724-731`); otherwise the error is dropped and the operation + times out. + - C# `CultMeshContentSessions.OnError` (`:256-260`) fails every pending request on any error. +8. **The reference is inconsistent with itself.** `src/GameCult.Networking/CultNetDatabaseServer.cs:560` + refuses a variant change with a codeless error. The subscription server refuses the same condition with + `ForVariantUnsupported` (`CultNetDatabaseSubscriptionServer.cs:211/333/366`). +9. **routingHint is never consumed.** No receiver in any runtime reads it. C# builds it at + `CultNetDatabaseServer.cs:638-650`, and Rust's decode discards it. +10. **Consumer audit** (read-only, all of `F:\Projects` except CultLib; no git in Aetheria): + - Outside CultLib, nothing sets `code`, `details` or `routingHint`, and nothing branches on a code. + - Emitters: + - Huginn: `crates/huginn-daemon/src/serve.rs:288/307/312`, Rust struct literals + `Error { error, code: None, details: None }`, pinned at cultnet-rs `8fc74c70`. + - AetheriaEve: `Aetheria.State.Daemon/Program.cs:1737/1778`, C# object initializers. + - Decoders read `.error` only: Gjallar `Program.cs:574`, Mimir + `MimirOdinMoveProofEvidenceRingProvider.cs:80`, Hermodr `hermodr-daemon.cjs:1380` (hand-rolled), + AetheriaEve's smoke test (text match against its own emit text), and Huginn's tests (text match). + - Stale vendored TS: + - `StreamPixels/vendor/CultLib` carries the current closed schema, so any change here breaks its + TS error decode until it is re-vendored. + - `Heimdall/vendor/CultLib` carries a pre-R-X schema with `additionalProperties:false` and no + `routingHint`, so it already refuses every error C# sends today. + - Odin, Idunn, Muninn, Ghostlight and voidbot: zero hits. + +## 1. Ends + +- **One schema.** `contracts/cultnet/cultnet.error.schema.json` is the only authority on the shape and on the + code registry. C# embeds it (`GameCult.Networking.csproj:19`), TS mirrors it at build time, and Rust + vendors a copy held byte-equal by test. Nothing else publishes a `cultnet.error.v0` shape (F-1 covers + Python). +- **Every runtime emits only schema-valid errors.** This is proven per emit path against the schema, and + byte-equal to the C# reference for every registry code. +- **Every runtime decodes every error it can receive into a typed value.** A known code becomes its typed + variant with its typed details. An unknown code, if Q1 says so, becomes a typed "other" that keeps the + code string and the raw details. No well-formed error is ever refused, and no field is silently dropped + (routingHint, messageId). +- **Code and details have one owner.** The details a message carries are decided by its code. The types in + every runtime make "cursor_stale with a field" unrepresentable. No free-standing code string plus a + grab-bag details object. +- **A refusal is an answer, not a silence** (R-2 precedent, and Q4). A server that refuses a peer's request, + and can still send, sends a coded error that correlates to the request. +- **The public surface is complete in every runtime**, per the consumer-expectation doctrine: + - a typed error value; + - a constructor per registry code; + - a typed exception or error that request APIs raise when the peer refuses, carrying the typed value. + +## 2. Identity, lifecycle and authority + +| Kind | What names it | Lifecycle | Who decides | +|---|---|---|---| +| Error message | `schemaVersion: cultnet.error.v0`; correlated to its request by `messageId` (Q3) | Sent once, in reply to one request or subscription. Never persisted. | The server that refused. | +| Error code | A snake_case string in the schema's registry (`$defs/knownCode`) | Added by one coordinated change: schema, C# constructor, a C#-written vector, and a typed variant in every runtime. Never renamed and never reused. A retired code stays in the registry marked retired, so an old reader still decodes it. | The operator rules meaning; Self keeps the registry. | +| Details for a code | The code | Frozen with the code. Adding a key to an existing code's details is a new code (or a v1), never an in-place edit (see Q2). | Same as the code. | +| Unknown code, at a reader | The raw string | Preserved as typed `Other`. The reader acts on the `error` text and the absence of a known code. | The reader (Q1). | +| Vector file | `contracts/cultnet/interop/error-vectors.cs-written.json` | Regenerated only by the C# writer test under `CULTNET_WRITE_VECTORS=1` and committed. Every other runtime judges it read-only. | C#, the reference. | +| routingHint | Shard authority refusal | Sent with a null code (unchanged). Typed in every decoder. | CultNetDatabaseServer's authority path. | + +## 3. Operator questions + +**Rulings (operator, 2026-09-30):** "Q1: B / Q2: B / Q3: A / Q8: A". +- **Q1:** closed for emitters, open for readers. An unknown code decodes to a typed `Other`. +- **Q2:** per-code details. +- **Q3:** a nullable top-level `messageId`. +- **Q8:** v0 is corrected in place, with the semver bumps the policy requires. +- Q4-Q7 are still open. +- Cut 1 waits for `hands/cultmesh-put-serve-bound` to merge. + +### Q1. What does a reader do with an error code it does not know? + +Today Rust and TypeScript refuse the whole message. The probes above show the result: the peer's error text +is lost and the caller sees "not a recognized code". In cultmesh-rs's snapshot client the `?` at `lib.rs:812-815` +even turns a peer refusal into a local decode failure. So under a closed enum every new code breaks every +reader that has not upgraded. By `docs/semver-policy.md:86-91` that makes each new code a breaking wire +change, and nine codes are already in flight. + +- **A. Closed enum (today).** Readers refuse unknown codes. Every code addition is a coordinated, + semver-major wire change across all runtimes and consumers. +- **B. Closed for emitters, open for readers.** The schema keeps the registry, and every runtime's emit + constructors can produce only registry codes (tests prove each emit path valid against the schema). + Readers decode an unrecognized code (matching `^[a-z][a-z0-9_]*$`) into a typed `Other { code, details }`. + A new code is an additive, minor change: old readers still surface the text and know "an error I cannot + classify". This is how gRPC status codes and protobuf open enums behave. +- **C. Fully open.** No registry; any string, emit and decode. + +**Recommended: B.** A refusal must never fail to arrive because it is more specific than the reader +expected. B keeps the registry as the single meaning authority. C loses the meaning authority, and A makes +the error channel the most brittle message in CultNet. + +Depends on this: schema `code` shape, every decoder, the doc text at `cultnet-distributed-database.md:107-115`. + +### Q2. How are details shaped on the wire? + +Today C# writes all four detail keys (`field`, `value`, `asOf`, `current`), nil included, because +`CultNetErrorDetails` (`CultNetSchemaMessages.cs:592-603`) is a MessagePack map-mode object. The codes in +flight need new keys: `responseBytes`, `maxSnapshotBytes`, `documentCount`, `maxSnapshotDocuments` and +`schemaVersion`. + +- **A. One shared details record; grow it.** Every detail key of every code sits on one record, written as + nil when unused. Each new key changes the bytes of every existing error that has details, so all of the + C# vectors must be recaptured on every addition. It also breaks every strict reader: TS Ajv, and + StreamPixels' vendored schema, have `additionalProperties:false`. And the type cannot tie a key to its + code. +- **B. Per-code details.** `details` carries exactly the keys its code defines. A defined-but-optional key + is present as nil; another code's keys are absent. Keys are named for their meaning, which retires the + `field: "schemaId"` / `field: "subscriptionId"` indirection that exists only because the record is + shared. Adding a code never changes an existing code's bytes. In every runtime, code and details are + fused into one tagged type: + - Rust: an enum with data. + - C#: sealed subclasses, with a custom formatter for `CultNetErrorMessage`. There is precedent in + `src/GameCult.Mesh/CultMeshRouteRecordFormatter.cs`. + - TS: a discriminated union. + - Kotlin: a sealed class. + - Python: constructors per code over a frozen dataclass. + + The churn is one time. The five existing C# cases with details change bytes, and all eight Rust hex + cases move into the shared vector file anyway (Cut 1). +- **C. Opaque string map.** Untyped; no runtime can offer a typed accessor. + +**Recommended: B.** A is the churn the brief warned of, and it recurs on every future code. B pays it once, +at the moment every vector is being recaptured regardless. + +Depends on this: the schema's `details`, the C# model and formatter, `encode_error_details` +(`packages/cultnet-rs/src/contracts.rs:1339-1366`), and every decoder. + +### Q3. Does an error carry the `messageId` of the request it refuses? + +Python already sends `messageId` (`server.py:583`); the schema forbids it. Without it: + +- `cultmesh-browser` drops an error whenever two operations are pending (`index.ts:724-731`); +- C# content sessions fail every pending request on any error (`CultMeshContentSessions.cs:256-260`). + +Adding a top-level key changes the bytes of every C# error. That happens once, in the same recapture as Q2. + +- **A.** Add a nullable top-level `messageId`. A reply to a request echoes the request's id, and an error + with no request (a subscription push, an unsolicited refusal) sends nil. Receivers correlate by it and + fall back to their current behavior when it is nil. +- **B.** No correlation. Python deletes the field. + +**Recommended: A.** Correlation is a capability any multiplexing client needs, and one runtime already +depends on it. + +### Q4. Do servers send `document_unservable` (and every other refusal they can) to the peer? + +Self ruled that "unservable" must be distinguishable as a typed code in every runtime. On the branch: +Python sends it coded, TypeScript sends it codeless, and Rust tells only its local caller, so the Rust +peer times out. The same Rust silence covers every other document-server rejection: a sink refusal, a +failed snapshot source, too many snapshot documents, and a snapshot response that is too large. + +- **A.** Every refusal of a peer's request that the server can still send is answered with a coded error + carrying the request's `messageId`, and is also reported locally (Rust's `ApplicationRejected` stays as + the local report). The refusals that cannot be answered are the ones where the answer itself cannot be + queued: Rust `PayloadBudgetFull` and `ResponseQueueFailed`. Those stay local-only, and the doc says so. +- **B.** Local-only in Rust; peer replies only where a runtime already sends them. + +**Recommended: A.** R-2 already ruled that a silence is worse than a missing feature +(`docs/cultnet-selection-cut.md:2049-2053`), and the operator's ruling that CultMesh "refuses" puts only +means something to the party that sent the put. + +### Q5. Does Python's `unregistered_document_put` / `unregistered_document_delete` become `unowned_schema`? + +Python sends these when the node has no registered document type for the record's schemaId +(`cultmesh_py/node.py:653-666`). C# `unowned_schema` means "no shard owns this schema and key" +(`CultNetSchemaMessages.cs:537-544`). To a peer both mean "this server will not take that schema here; route +elsewhere". The put and delete variants duplicate what the peer already knows (it sent the message). + +- **A.** Both collapse into `unowned_schema { schemaId }`. The registry text broadens to "this server + stores no document for that schema (and key)". +- **B.** Join as a separate `schema_unregistered { schemaId }`. +- **C.** Join both codes as they are. + +**Recommended: A.** It is one meaning at the peer, and there is one action to take. + +### Q6. Does Python's `malformed_document_put` / `malformed_document_delete` become one `message_invalid { field }`? + +These are sent when the request lacks a required field (`server.py:355-359`, `383-387`). C# has the same +condition, codeless, at `CultNetDatabaseServer.cs:373/415` and `CultNetDatabaseSubscriptionServer.cs:148` +("requires a subscriptionId or messageId"). + +- **A.** One `message_invalid { field }`, naming the missing or ill-typed field. It is used by Python's four + sites and C#'s three. +- **B.** Join both per-message codes and leave C# codeless. + +**Recommended: A.** The per-message suffix repeats the request's own schemaVersion, and C# gains a code for +free. + +### Q7. Does `simulation_observations_disabled` become `unsupported_schema_version`? + +Python sends it when the observation hub is off (`server.py:593-598`), with +`details { schemaVersion: "cultnet.simulation_observation.v0" }`, which is exactly the +`unsupported_schema_version` shape. cultmesh-ts's "no cache for snapshot requests" and "no operation +handler" refusals (`index.ts:5875`, `5913`) mean the same thing: this server does not serve that message. + +- **A.** Collapse. `unsupported_schema_version` means "this server does not serve messages of this + schemaVersion: unknown, a newer version, or not enabled here". The three TS sites and Huginn's "answers + only X, Y, Z" adopt it. +- **B.** Keep a separate `feature_disabled`-style code. + +**Recommended: A.** A peer cannot act differently on "not enabled" and "not known". Python's +`verify.py:335` check follows the rename. + +### Q8. Is this a correction of `cultnet.error.v0` in place, or a new `cultnet.error.v1`? + +Under Q2-B and Q3-A, strict v0 readers (TS Ajv and vendored copies) refuse the new bytes. Tolerant readers +(C#, Rust, Python) keep working, because the probes show they ignore unknown keys and treat a missing key +as null. + +- **A. In place.** The v0 contract has never been decodable across all five runtimes (section 0). No + consumer outside CultLib reads code or details (finding 10). Every publishing package takes the semver + bump its policy requires: + - `org.gamecult.cultlib` major, because the public `CultNetErrorDetails` properties change; + - `cultnet-py` and `cultmesh-py` major, because emitted codes change; + - Rust pins move deliberately. + + StreamPixels and Heimdall re-vendor. +- **B. v1.** Add `cultnet.error.v1`, keep v0 emit for peers that did not advertise v1 in hello, and run + two error encoders in every runtime until v0 retires. + +**Recommended: A.** B builds a version-negotiation path for a message nobody outside CultLib parses beyond +`.error`. The one thing every old reader does need, the `error` text, survives A in every tolerant runtime. + +## 4. The registry (as recommended; rewrite on each ruling) + +| Code | Meaning | Details (per Q2-B; `?` = present-as-nil when absent) | Emitted by after this map | +|---|---|---|---| +| `selection_invalid` | A selection field failed the door. | `{ field, value? }` | C#, Rust (selection) | +| `cursor_stale` | The cursor predates the server's retained history. | `{ asOf, current }` | C#, Rust | +| `cursor_invalid` | The cursor could not be parsed. | none (details nil) | C#, Rust | +| `reference_outside_target` | A hop reference left the declared target. | none | C#, Rust | +| `variant_unsupported` | The selection selects a document variant CultNet does not carry. | `{ subscriptionId? }` | C# (incl. `DatabaseServer.cs:560`) | +| `unowned_schema` | This server stores no document for that schema (and key). | `{ schemaId }` | C#, Python (Q5) | +| `unsupported_schema_version` | This server does not serve messages of this schemaVersion. | `{ schemaVersion }` | Python, TS, Huginn (Q7) | +| `message_invalid` | The request lacks a required field or has an ill-typed one. | `{ field }` | C#, Python (Q6) | +| `snapshot_document_limit_exceeded` | The snapshot answer would exceed the document bound. | `{ documentCount, maxSnapshotDocuments }` | Python, Rust (Q4) | +| `snapshot_byte_limit_exceeded` | The snapshot answer would exceed the byte bound. | `{ responseBytes, maxSnapshotBytes }` | Python, Rust (Q4) | +| `document_unservable` | A put was refused because no snapshot could ever return the stored document under the byte bound. | `{ responseBytes, maxSnapshotBytes }` | Python, TS, Rust (Q4) | + +A null code is legal and means "unclassified". Examples: an exception message from a failed snapshot +source, an authority refusal carrying `routingHint`, and "not authorized" +(`CultNetDatabaseSubscriptionServer.cs:165`). This map does not invent codes for those. + +`snapshot_byte_limit_exceeded` and `document_unservable` have identical details. That is deliberate, and it +is the fixture trap named in the verification rules. + +--- + +## 5. Cuts + +All cuts are made on one campaign branch, `hands/error-contract`, from `main` after +`hands/cultmesh-put-serve-bound` merges. The branch merges to `main` once, after Cut 5's Soul pass. The +reason: Cut 1 changes C# bytes, and Rust's static hex cases would stay green against the stale bytes +(finding: `tests/error_message.rs` pins constants, not the reference). A partial merge would ship a +wire split while every suite passed. Each cut gets its own worktree, created by Self. Heavy verification +runs through `ygg-verify.sh`, never on Starfire. + +### Cut 0. Unblock the interop lane + +- **Repo/branch:** CultLib, `hands/interop-guard` from `main`, merged alone ahead of the campaign. It + does not depend on the rulings. +- **Deletes first:** nothing. +- **Per-file changes:** `.github/workflows/cultnet-interop.yml:72`. The `$stray` filter also accepts + `^packages/cultnet-rs/contracts/cultnet/`, with a comment naming R-AB and + `packages/cultnet-rs/tests/schema_discovery_contracts.rs:74-99` as the equality guard. The namespace + check is unchanged. +- **Authority map:** no ownership change. The canonical owner stays `contracts/cultnet/`, and the Rust + mirror is derived from it, with drift judged by the Rust test. +- **Verification:** + - Push the branch and run the workflow. Report every step that goes red after the guard. None of those + steps has run on main for 40 runs, so expect more red, and treat each failure as its own finding; + do not fix it in this cut. + - Negative: edit one byte of a vendored copy locally and check that the Rust equality test fails. + Reach it on Yggdrasil with `cd packages/cultnet-rs && cargo test --test schema_discovery_contracts`. +- **Ledger:** +3 lines. + +### Cut 1. The contract and the C# reference + +- **Repo/branch:** CultLib `hands/error-contract`, worktree named by Self. +- **First:** capture the current C# bytes for all eight factory cases, plus `cs_codeless` and `cs_routing`, + into the Hands report. This is the before-image a Soul pass compares against. +- **Deletes first:** + - `CultNetErrorDetails` (`src/GameCult.Networking/CultNetSchemaMessages.cs:588-603`, 16 lines) as a + shared record. + - The `field`/`value`/`asOf`/`current` property block and the `[MessagePackObject]`/`[Key]` attributes + on `CultNetErrorMessage` (`:497-521`). Its serialization moves to one formatter. + - `SoulP4dProbeTests.Wire_UnownedSchemaBytes` (`tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs:191-199`). + It is an `[Explicit]` hex printer, superseded by the vector writer. + - The closed-enum paragraph at `contracts/cultnet/cultnet-distributed-database.md:107-115` and + the `code`/`details` descriptions in the schema (`:28-42`). +- **Adds:** + - **Schema** (`contracts/cultnet/cultnet.error.schema.json`, rewritten; the Rust mirror is re-copied + byte-equal): + - Top level: `schemaVersion`, `error` and `routingHint` as today, plus `messageId: string|null` (Q3) + and `code`. All five are required, present-as-nil (R-X is unchanged at the top level). + - `$defs/knownCode` is the registry enum from section 4. + - `code` is `null`, or a string matching `^[a-z][a-z0-9_]*$` (Q1-B). + - `details` is decided by a top-level `oneOf`, one branch per known code, each fixing `code` by + `const` and `details` by exact `properties`/`required`/`additionalProperties:false`. Two more + branches: a null code with null details, and an unknown code (`not: {$ref knownCode}`) with details + `object|null`, open. + - Use `oneOf`/`not`/`const` only. `tests/GameCult.Networking.Tests/MiniJsonSchemaValidator.cs:30-35` + implements those and **not** `if`/`then`/`allOf`, and it refuses unimplemented keywords. TS + compiles with Ajv2020 (`packages/cultnet-ts/src/contracts.ts:1,674`), which handles both. + - **C# model** (`GameCult.Networking`): + - `CultNetErrorMessage { SchemaVersion, Error, MessageId?, RoutingHint?, Refusal? }`, where + `Code => Refusal?.Code` is derived and not settable. + - `abstract CultNetRefusal { string Code }`, with one sealed subclass per registry code carrying its + typed details (`CultNetCursorStaleRefusal(ulong AsOf, ulong Current)` and so on), plus + `CultNetUnknownRefusal(string Code, IReadOnlyDictionary? Details)` for Q1-B. + - `CultNetErrorCodes`: string constants, one per registry code. + - `CultNetErrorMessageFormatter : IMessagePackFormatter`. It writes the five + top-level keys in order (schemaVersion, error, messageId, routingHint, code, details), and details + as exactly the refusal's keys. It reads by key name, tolerates unknown keys at both levels, and + maps an unknown code to `CultNetUnknownRefusal`. Register it through `[MessagePackFormatter]` on + the type, the way `CultMeshRouteRecordFormatter` is wired. + - Factories: keep `ForSelectionInvalid`, `ForCursor`, `ForReferenceOutsideTarget`, + `ForVariantUnsupported` and `ForUnownedSchema` as thin constructors over the refusal types, and add + one per new code. Each takes an optional `messageId`. + - `CultNetPeerErrorException : IOException`, carrying the `CultNetErrorMessage`. + - **Vector writer:** `ErrorVectorTests.WriteErrorVectors`, gated on `CULTNET_WRITE_VECTORS=1` like + `SelectionParityVectorTests.cs:460-552`. It writes `contracts/cultnet/interop/error-vectors.cs-written.json`: + one entry per registry code (both optional-key states where a key is optional), `cs_codeless`, + `cs_routing`, one unknown code with open details, and one error with `messageId` set. Each entry + holds `{ name, messagePackBase64, expected: { error, messageId, code, details, routingHint } }`, + where `expected` is a JSON projection every runtime compares its typed decode against. +- **Per-file changes** (`8fc74c70`): + - `CultNetDatabaseServer.cs:560`: `ForVariantUnsupported(refusal.Message, subscriptionId)`. + - `CultNetDatabaseServer.cs:373`, `:415` and `CultNetDatabaseSubscriptionServer.cs:148` become + `message_invalid { field: "subscriptionId" }` (Q6-A). + - Every `SendCultNet(CultNetErrorMessage...)` that answers a request passes the request's `MessageId`. + - Each receiver listed in finding 6 throws `new CultNetPeerErrorException(error)` instead of + wrapping `error.Error`. `CultMeshContentSessions.OnError` (`:256-260`) fails only the pending entry + whose id matches `error.MessageId`, and falls back to all entries only when the id is nil. + - `CultNetSchemaContractTests.cs:165-214`: the per-factory tests are replaced by one test that validates + every vector's decoded JSON against the schema with `MiniJsonSchemaValidator`, and one test that + serializes each registry factory and asserts byte equality with its committed vector. + - Rewrite `contracts/cultnet/cultnet-distributed-database.md` "Error Codes" to the registry, and state + the Q1 and Q2 rules. +- **Authority map:** + - Owner: the schema's registry owns code meaning and detail shape. `CultNetErrorMessageFormatter` owns + C# bytes. The refusal subclass owns the code-to-details binding. + - Inputs: the refusal instance, messageId, routingHint. + - Outputs: bytes, and `cs-written.json` for the other runtimes. + - Derived state: `CultNetErrorMessage.Code` becomes derived; the refusal decides it. + - Forbidden writers: any `Code = "..."` assignment (it cannot compile); `[Key]` attribute serialization + of the message; hand-typed hex anywhere. + - Shared paths: every server send uses the factories; every receiver uses `CultNetPeerErrorException`. + - Deletion line: `CultNetErrorDetails` and the settable `Code`. +- **Verification** (`dotnet` image): + - Builds: `tests/GameCult.Networking.Tests`, `tests/GameCult.Mesh.Tests`. + - `ErrorVectorTests` with `CULTNET_WRITE_VECTORS=1`, then without, and check that the committed file + reproduces byte-for-byte. + - Tests and the rules they pin: + - vector round trip: bytes, decode, bytes are equal for every entry; + - a registry factory is byte-equal to its vector; + - every vector validates against the schema; + - an unknown code decodes to `CultNetUnknownRefusal` with its raw details intact; + - a Python-shaped error (`py_snapshot_byte_limit` above, with the Q2/Q3 shape) decodes typed with + `responseBytes` and `maxSnapshotBytes` intact. Today's C# silently empties those. + - content-session correlation: two pending requests, an error for the second, and the first still + completes. + - Mutation: Stryker.NET on the formatter and factories. The survivors that must die: + - swapping two known codes whose details are identical (`snapshot_byte_limit_exceeded` and + `document_unservable`); + - padding a nil key into details; + - dropping `messageId` on encode; + - mapping an unknown code to null. + - Negative greps (checked at HEAD, where they hit the sites being cut): + - `git grep -nE 'new (IOException|InvalidOperationException)\(error\.Error\)' -- src` must return 0 + (9 hits at HEAD). + - `git grep -nE 'Field = "(schemaId|subscriptionId)"' -- src` must return 0 (2 at HEAD). + - Operator: none. +- **Ledger:** about -60 C# (the details record, the settable code, the probe printer, and the per-factory + schema tests), +250 (the formatter, the refusal types, the vector writer). The schema grows by about 90 + JSON lines. The net is positive, bought by Q1/Q2 typing in the reference. + +### Cut 2. Rust: cultnet-rs and cultmesh-rs + +- **Deletes first:** + - `CultNetErrorCode` and `CultNetErrorDetails` as separate types (`packages/cultnet-rs/src/contracts.rs:252-310`, + 59 lines); + - `require_error_details` (`:1121-1146`); + - `encode_error_details` (`:1339-1366`); + - the eight hex `Case`s and the header comment in `packages/cultnet-rs/tests/error_message.rs:1-160`. + That file is rewritten to read the vector file. + - Keep `require_legacy_optional_u64` (`:1149`). Check with `git grep` for other callers first; at HEAD, + only the error details use it. +- **Adds:** + - `CultNetMessage::Error { error: String, message_id: Option, routing_hint: Option, refusal: Option }`. + - `enum CultNetRefusal`, with one variant per registry code and its typed fields, plus + `Other { code: String, details: Option }`. + - `CultNetRefusal::code(&self) -> &str`. + - A constructor `CultNetMessage::error(text)` and builder-style `with_refusal` / `with_message_id`, so + consumers stop writing struct literals. Huginn has three. + - Decode reads every top-level key by name and routes unknown codes to `Other` (Q1-B). + - Encode writes the Cut 1 key order. + - `CultNetShardRoutingHint` is typed, reusing the crate's existing shard descriptor type. +- **Per-file changes:** + - `packages/cultnet-rs/src/contracts.rs:361-368` (variant), `:1097-1115` (decode), `:1300-1336` (encode). + - `packages/cultnet-rs/src/selection.rs:1210-1255`: the four refusals build `CultNetRefusal` variants. + - `packages/cultnet-rs/src/content.rs:316`: uses `CultNetMessage::error`. + - `packages/cultmesh-rs/src/lib.rs:812-827`: a peer error becomes a typed error carrying the refusal, + instead of `anyhow!` text. Name it `CultMeshPeerError`, next to the crate's existing error types. + - `packages/cultmesh-rs/src/rudp_document_server.rs` (Q4-A): each `CultMeshRudpRejectionReason` that + can be answered sends the matching coded error, correlated by the request's message id, before + returning `ApplicationRejected`. `PayloadBudgetFull` and `ResponseQueueFailed` stay local. +- **Authority map:** + - Owner: `CultNetRefusal` owns code and details. The schema owns the registry. + - Forbidden writers: string codes (`from_wire_str` survives only inside decode); struct literals of + details. + - Shared paths: selection refusals, document-server refusals and content errors all go through the one + encoder. +- **Verification** (`rust` image, `cd packages/cultnet-rs` then `cd ../cultmesh-rs`): + - `cargo test --workspace`. + - `tests/error_message.rs` pins: every vector decodes to `expected`; encoding the decoded value + reproduces the vector's bytes; the unknown-code vector decodes to `Other`; the routing vector keeps + `routing_hint`. + - cultmesh-rs: a put over the bound makes the peer receive `document_unservable` with the request's + message id, and details equal to the computed size and the bound. Run it at two bounds so that a + constant or an offset in `maxSnapshotBytes` dies. + - A peer that sends a Python-shaped `snapshot_byte_limit_exceeded` produces a `CultMeshPeerError`, not + a decode failure. + - Mutation: `cargo mutants --in-diff` on the cut's diff. Triage every survivor by name. + - Negative: `git grep -n 'CultNetErrorDetails' -- packages` must return 0. +- **Consumer note:** Huginn's `serve.rs:288/307/312` and its tests stop compiling at the next pin bump. + The migration is mechanical (`CultNetMessage::error(...)`), and `:312` adopts `unsupported_schema_version` + (Q7). Self routes it as a Huginn change; it is not part of this branch. + +### Cut 3. TypeScript: cultnet-ts, cultmesh-ts and cultmesh-browser + +- **Deletes first:** + - the two-field `CultNetErrorMessage` interface (`packages/cultnet-ts/src/contracts.ts:167-170`); + - the four literal error constructions in `packages/cultmesh-ts/src/index.ts:5875-5878`, `5885-5888`, + `5913-5916` and `5922-5925`, and the branch's codeless unservable site (`0eb7434a`); + - `rejectCorrelated`'s single-pending heuristic (`packages/cultmesh-browser/src/index.ts:724-731`). +- **Adds:** + - `CultNetErrorMessage` with every field. + - `type CultNetRefusal` as a discriminated union on `code`, with an `{ code: string; details: Record | null; known: false }` arm. + - `createCultNetError(text, { messageId, refusal })` as the only constructor. + - `class CultNetPeerError extends Error { message: CultNetErrorMessage }`. + - `parseCultNetMessage` keeps Ajv validation against the canonical schema; it now accepts every schema + branch, including unknown codes. +- **Per-file changes:** + - cultmesh-ts: the three "does not serve this" sites (`5875`, `5913`, `5922`) send + `unsupported_schema_version` (Q7-A); the snapshot catch (`5885`) sends a null code; the unservable site + sends `document_unservable` with details. + - cultmesh-ts `6527-6529` throws `CultNetPeerError`. + - cultmesh-browser `163`, `584` and `690`: correlate by `messageId`, falling back to today's behavior only + for a nil id. + - `packages/cultnet-ts/src/idunn-odin-presence-publisher.ts:118-120` throws `CultNetPeerError`. + - The legacy `gamecult.networking.v0` path (`contracts.ts:840-844`, `:899-900`) is unchanged. That is a + separate legacy contract. +- **Verification** (`kotlin` image, which carries Node 24; `npm install`, then build cultcache-ts before + cultnet-ts): + - A new `test/error-vectors.test.ts` pins that every vector parses and deep-equals `expected`, and that + `encode(createCultNetError(...))` is byte-equal to each registry vector. + - The cultmesh-ts document-server tests assert that each refusal parses with TypeScript's own + `parseCultNetMessage`. That is the probe's `ts_two_key` failure, pinned. + - Browser: two pending operations, an error for the second, and the first still resolves. + - Mutation: StrykerJS on the diff. + - Negative: `git grep -nE 'schemaVersion: "cultnet\.error\.v0",' -- packages/cultmesh-ts/src packages/cultmesh-browser/src` + must return 0. Exclude `cultnet-ts/src/schema-discovery.ts:302` (a descriptor) and + `contracts.ts:842` (legacy decode); both are legitimate hits for the unscoped pattern. + +### Cut 4. Python: cultnet-py and cultmesh-py + +- **Deletes first:** + - `_error_response` (`packages/cultmesh-py/src/cultmesh_py/server.py:573-589`); + - `unsupported_schema_version_error` (`packages/cultnet-py/src/cultnet_py/interop_peer.py:409-417`); + - every string literal code in `server.py` (lines in finding 3). +- **Adds:** + - `cultnet_py/errors.py`, holding: + - `CultNetErrorCode(StrEnum)`, the registry; + - a frozen `CultNetError` dataclass (`error`, `message_id`, `routing_hint`, `code: CultNetErrorCode | str | None`, + `details: Mapping | None`); + - one constructor function per code, taking exactly that code's keys; + - `to_wire()`, which writes the five top-level keys in the Cut 1 order, nil included; + - `parse_cultnet_error(dict)`. + - `CultNetPeerError` (`client.py:46-50`) keeps `.response` and gains `.error: CultNetError`. +- **Per-file changes:** + - Sites `server.py:158-163`, `355-394`, `555-569` and `593-598` and the branch's `document_unservable` + site use the constructors, with the Q5-A/Q6-A/Q7-A codes. + - `interop_peer.py` uses the constructor. + - `verify.py:335` reads `error.error.code`. + - Tests `packages/cultmesh-py/tests/test_cultmesh.py:2088`, `3791`, `4323` and `4363-4371`, and + `packages/cultnet-py/tests/test_cultnet.py:2342`, follow the renames. +- **Verification** (the image the campaign already uses for Python; state it in the report): + - A vector test pins that every vector parses to `expected` and that each constructor's `msgpack.packb` + is byte-equal to its vector. + - Mutation: mutmut on `errors.py` and the changed server sites. + - Negative: `git grep -nE '"schemaVersion": "cultnet\.error\.v0"' -- packages/cultnet-py/src packages/cultmesh-py/src` + returns only `errors.py`. +- **Not here:** F-1 (`schema_catalog.py` publishes its own shapes). + +### Cut 5. Kotlin: cultmesh-kotlin + +- **Adds:** + - A `CultNetErrorMessage` data class and a `sealed class CultNetRefusal` (known codes plus `Other`), + with a `CultNetMessage.asError(): CultNetErrorMessage?` view. + - `class CultNetPeerException(val error: CultNetErrorMessage) : IOException`. + - At `CultMesh.kt:1278` and `:1334`, and at every `receiveSchemaMessage` consumer that expects a + response, an error response throws `CultNetPeerException` before the `require`. +- **Verification** (`kotlin` image, `pwsh -File build.ps1`): + - A vector test pins that every vector decodes to `expected`. + - Kotlin emits no errors today, so encode parity applies only if a Kotlin server path is added. Say + so in the report; do not invent one. + - There is no mutation tool. Behavioural tests cover it: an error reply to a schema-catalog request + raises `CultNetPeerException` carrying the code. + +### Cut 6. Merge, release, consumers + +- Soul runs over the whole branch before the merge: + - rerun every vector suite; + - decode each runtime's live emissions with every other runtime. The interop lane from Cut 0 is the + harness; add an error round to `tests/GameCult.Networking.InteropPeer` and + `packages/cultnet-ts/test/interop` if Soul finds a runtime pair unreached. +- Release per `docs/semver-policy.md`, with a Soul pass before tags: + - `org.gamecult.cultlib` major, and the tracked Unity DLLs rebuilt, since they embed the schema; + - `cultnet-py` and `cultmesh-py` major. +- Consumers, routed by Self as separate repo changes: + - Huginn: pin bump plus Cut 2's note. + - StreamPixels and Heimdall: re-vendor. Heimdall is already broken by R-X. + - AetheriaEve, Gjallar and Mimir compile against the live CultLib path and read only `.Error`, so they + need no change. That is inferred from the audit; the next build of each confirms it. + - Hermodr: none. + +## 6. Subtraction ledger + +| Cut | Removed (estimate) | Added (estimate) | Formats / targets | +|---|---|---|---| +| 0 | 0 | 3 | none | +| 1 | ~60 C#, schema text | ~250 C#, ~90 schema | + `error-vectors.cs-written.json`; - `[Explicit]` hex printer | +| 2 | ~150 (types, two helpers, 160-line hex test) | ~200 | hex constants gone; vectors read from file | +| 3 | ~40 | ~120 | none | +| 4 | ~40 | ~120 (`errors.py`) | 8 ad-hoc codes → registry | +| 5 | 0 | ~80 | none | + +The net is positive, by roughly 500 lines, across five runtimes. It buys typed decode where there was +none (Kotlin, TS and Python), a correlation capability, and one vector file in place of hand-copied hex. +The pressure point: if Cut 1's formatter passes 150 lines, report it rather than trimming typing. + +## 7. Follow-ups outside this map + +- **F-1.** `packages/cultnet-py/src/cultnet_py/schema_catalog.py` publishes about 350 lines of hand-built + schemas under canonical `$id`s. Python should package and serve `contracts/cultnet/*.schema.json`, the + way Rust vendors them. It is a separate cut, because it covers every message, not errors. It can wait + because the catalog is advisory: no runtime validates a peer's message against a published descriptor. +- **F-2.** Codes for the remaining codeless refusals (authorization, failed snapshot sources, routing). + Wait for a consumer that branches on them. +- **F-3.** Whatever the Cut 0 lane reveals once its later steps run again. diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md index dfdeef1f..e7c8f6ae 100644 --- a/docs/cultnet-rudp-ack-cut.md +++ b/docs/cultnet-rudp-ack-cut.md @@ -622,6 +622,12 @@ The ack cuts build on these behaviours and do not alter them: drops what it owed. How a new generation is recognised (Cut 1c) needs one (Q-A6). The two are separable because 1b does not forget peer state: the peer may not know the session ended, and forgetting what was received would let its retransmits be delivered twice. + **Correction (Cut 1/1b Soul, 2026-09-30): they are separable as specs, not as merges.** 1b drops owed + writes, which leaves holes the sender will never fill; the receiver keeps `R` across the ending, so if the + next handshake does not reset it, every later ordered frame is held and acked, and flush succeeds while + nothing is delivered. Only 1c's reset on a non-repeated handshake removes that. **1b merges only together + with 1c.** Soul also found `R` seeded by the first reliable packet rather than the handshake; ruled: + `R` is seeded only by the handshake, and reliable data before the Accept is neither acked nor delivered. - **What must hold after both.** A write is Acknowledged or Invalidated within the generation it was issued in. A restarted client is admitted. A new server session's frames are delivered after a reconnect. A retransmitted Connect is still a repeat. @@ -1543,3 +1549,47 @@ Each question stands alone. `packetsDropped` until K1. Record the stats shape in the parity doc once both land. - **The P4 probe at the old pins** predates its fix; rerun `09318d13`'s P4 against a pinned sender if old-sender evidence is needed. + +**Follow-up (media FEC Cut 3 Hands, 2026-09-30):** `channel_send_options` in `rudp.rs` has no test for the +`"schema"` and `"latest"` channel arms. Deleting either falls to the default and every test stays green. Pin their send +flags on the wire the way the media arm is now pinned (raw peer socket), in whichever cut next touches channel +profiles. + +**Rulings after the Cut 1/1b/1c Soul pass (Self, 2026-09-30):** +- **Stale data after a reconnect.** A reliable sequence at or before the generation's seeded `R` (serial arithmetic + within the receive window) is a duplicate: acknowledged, not delivered. This is what §6/§8 already promised. +- **Stale Connect (RFC 5961's challenge ACK).** A Connect whose sequence precedes the current generation's Connect + within the receive window is stale: it is answered with an Ack naming the current generation and does not reset. + A client whose Connect gets no Accept before its connect attempt times out starts a fresh attempt with a newly + drawn initial sequence, so a restarted client whose random sequence lands in that window still connects. +- **Server mode.** A Connect from a different endpoint is never a repeat; the session predicate stays sequence-only. + (Hands' deviation 5 was rejected: pinned clients restarting on a new port were locked out.) +- **The random default belongs to the session.** A session draws its own initial sequence when none is set, so one + options object reused across sessions or reconnects never repeats a sequence. +- **Connect attempts (Self, 2026-09-30, after the Cut 1c fix-batch Soul pass).** While a Connect awaits its Accept, only an + Accept the client honours retires it; an Ack never does (an Ack naming the pending Connect had wedged the client for + good). A fresh attempt after the connect-attempt timeout uses exactly the abandoned sequence + (receive window − 1), + not a random draw: that keeps a late copy of the abandoned Connect inside the new one's stale window while escaping the + server's stale window (unless the server sits exactly there, in which case the next attempt escapes). Only a client's + first Connect draws randomly. +- **Recorded divergence (not this campaign's to fix yet).** Only Rust has a receiver receive-ahead window (1,024). TS, + C# and Python acknowledge a far-ahead reliable frame by name and hold it behind an ordered gap indefinitely, so a + stranded ordered channel is silent loss there and "still owed" in Rust. Map a receive window for the three runtimes. +- **Residuals after the final Cut 1c Soul pass (recorded 2026-09-30, low, not fixed):** + - A late copy of a Connect from two or more attempts back (reordered > ~3 s) still restarts the server; the +window−1 + rule keeps only the immediately previous attempt stale. Rust then times out and reconnects; TS/C#/Python hit the + receive-window divergence above (silent loss behind the gap). + - A client pinning `initial_sequence` that restarts while its old data is still in flight after the restarted Connect + can be matched to the old session's resent Accept (byte-identical; the client cannot tell). Pinning forfeits restart + detection; the random default closes it. Document on the option. + - Repeated failed attempts walk the sequence up 4095 per 3 s and clamp at `u32::MAX−1` after ~18-27 days on one session; + make exhaustion a hard error the caller sees (the clamp removal mutant survives). +- **Cuts 3 and D Soul pass (Self, 2026-09-30).** Cut 3: the 4 MiB bound counts every byte sent above the lowest + unacknowledged sequence, acked or not (the spec's wording; the first implementation counted only pending packets). + Cut D: a send failure is either permanent (the datagram can never be sent as built, e.g. `EMSGSIZE`) or transient; + a permanent failure is returned from a caller-directed send and, inside a poll, ends only that peer's session with a + typed reason; transient failures stay counted losses. Windows `ConnectionResetError` on receive is idle, never fatal, + in every server. Cut D's negative grep `send_packet(...)?;` is obsolete: the `?` now propagates encode errors only. +- **Recorded (liveness, not fixed):** a lowest reliable packet lost forever stalls the sender 1,023 sequences ahead + while the peer stays alive; session timeout doesn't end it. TCP ends such a connection after a retransmission + limit (R2). Decide with Cut F (abandonment) whether a reliable packet outstanding past a bound ends the session. diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index 5e191df8..bd340c87 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -369,6 +369,34 @@ stripped (probe). Other findings: refuses v3 also contains C2a, so no released reader strips ids. C0 and C1 are never released on their own. Raw and cross-runtime writers that cannot see ids stay sticky-only. Fix batch 4 is in Hands. +**C2a F2 ruled (operator, 2026-09-30, accepting Self's recommendation B): records nobody claims are carried +untouched.** A store may hold records under a schema id that no local type owns or lists as compatible, for +example after a rename that declared nothing. +- Both store kinds carry those records byte for byte under their own id. The catalog lists them as foreign + records. +- Writes to other records proceed. +- A write that would overwrite or relabel an unclaimed record is refused with a typed schema conflict. +- The single-file store's whole-view write stops relabelling such records onto local types. The directory + store stops refusing every commit while it holds one. +- Why: batches 8-9 showed that the danger is rewriting a record under a new identity (F1, F6), not an + undeclared old id. Batch 10's owner-beats-lister rule makes declaring an old id cheap anyway. +- Self's first lean, "refuse loudly" (A), is superseded. + +**C2a batches 9-10 (Soul pass on `fdb8a60a`, 2026-09-30): hold.** +- **F6, A-B-A lost update on the single-file store.** Ruled by Self: any write that changes a record's bytes or + schema id mints a later `storedAt`. +- **F8, batch 9 over-refused.** An owner beats a lister, so v1 and v2 coexist. +- **F9.** Compatible ids compare as a set. +- **F10.** Typed refusal errors in TS and Python. +- Batch 10 is in Hands. +- **F7 ruled (operator, 2026-09-30): retire `AsSchemaAlias`.** "Schema aliasing sounds like a really bad idea, I + support retiring it." + - C# refuses a second type for one schema, as TS, Python and Rust already do. There is one type per schema in + every runtime. + - The API, its registry path (`IsExactWireAlias`) and the tests that exercised it (7 Mesh, 2 Caching) are + deleted or rewritten. + - No consumer called it. + **Operator rulings, 2026-09-30:** - **Duplicate unique-index values do not load** ("Duplicate indices should not load"). This covers plain records too, so R6-for-indexes becomes one general rule: diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 8fc633cb..c9c76b06 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -5,7 +5,7 @@ import json import os import threading from pathlib import Path -from typing import Any +from typing import Any, Callable, TypeVar from .backing_store import ( CultCacheEnvelope, @@ -21,6 +21,7 @@ STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3" _READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS) _STORE_FORMAT_PREFIX = "cultcache.store." _PERSISTED_RECORD_SLOTS = 4 +_Contents = TypeVar("_Contents") class StoreUnreadableError(ValueError): @@ -56,10 +57,11 @@ class JsonLinesBackingStore: def pull_all(self) -> list[CultCacheEnvelope]: with self._lock: - if not self.path.exists(): + text = _read_store(self.path, lambda path: path.read_text(encoding="utf-8")) + if text is None: return [] envelopes: list[CultCacheEnvelope] = [] - for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + for line_number, line in enumerate(text.splitlines(), start=1): if not line.strip(): continue raw = json.loads(line) @@ -121,12 +123,11 @@ class SingleFileMessagePackBackingStore: def pull_all(self) -> list[CultCacheEnvelope]: msgpack = self._msgpack() with self._lock: - # The disk decides the header: a file that is gone or empty is not marked. + # An empty file is not a store (no writer leaves one), so it is decoded and refused. + # The disk decides the header: a file that is gone is not marked. self._format = STORE_FORMAT_VERSION - if not self.path.exists(): - return [] - data = self.path.read_bytes() - if not data: + data = _read_store(self.path, Path.read_bytes) + if data is None: return [] try: decoded = msgpack.unpackb(data, raw=False) @@ -261,13 +262,44 @@ def _encode_snapshot( return [format_version, catalog, records] +def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None: + """Reads a store file, or returns None when nothing is at its path. + + Only nothing is absent. A dangling symbolic link is something: reading it as + empty would let the writer replace the link and move the store off its volume. + Any other failure to reach the file stays an OSError. + """ + try: + return read(path) + except FileNotFoundError: + if path.is_symlink(): + raise + return None + + +def _describe_identity(value: Any) -> str: + """A record key or schema id as a refusal may show it: a string is an identity and is + named; anything else is described by its type, since the value is the store's.""" + if isinstance(value, str): + return repr(value) + return f"<{type(value).__name__}>" + + +def _describe_header(header: str) -> str: + """Echoes a header only in the shape cultcache.store.v; the bytes are the store's.""" + version = header[len(_STORE_FORMAT_PREFIX) + 1:] if header.startswith(_STORE_FORMAT_PREFIX + "v") else "" + if version.isascii() and version.isdigit(): + return repr(header) + return f"an unrecognised {_STORE_FORMAT_PREFIX}* header of {len(header.encode('utf-8'))} bytes" + + def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None: if not isinstance(decoded, list) or not decoded: return None if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] not in _READABLE_STORE_FORMATS: raise ValueError( - f"CultCache store format {decoded[0]!r} is not readable; this runtime reads {STORE_FORMAT_VERSION!r} and {STORE_FORMAT_ELEMENT_IDS!r} only. " - "The store needs a runtime that resolves document variants." + f"CultCache store format {_describe_header(decoded[0])} is not one this runtime reads; " + f"it reads {STORE_FORMAT_VERSION!r} and {STORE_FORMAT_ELEMENT_IDS!r} only" ) if decoded[0] not in _READABLE_STORE_FORMATS: return None @@ -291,8 +323,9 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None raise ValueError("CultCache persisted records must be MessagePack arrays") if len(raw_record) > _PERSISTED_RECORD_SLOTS: raise ValueError( - f"CultCache record {raw_record[0]!r} (schema {raw_record[1]!r}) has {len(raw_record)} slots; " - f"this runtime reads {_PERSISTED_RECORD_SLOTS}. The store needs a runtime that resolves document variants." + f"CultCache record {_describe_identity(raw_record[0])} (schema {_describe_identity(raw_record[1])}) " + f"has {len(raw_record)} slots, more than " + f"the {_PERSISTED_RECORD_SLOTS} of a {STORE_FORMAT_VERSION} record, so this is not a valid store" ) key, schema_id, stored_at, payload = raw_record if not isinstance(key, str) or not key: diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index a33eeb4b..b0f2be2a 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import tempfile import threading import unittest @@ -1321,6 +1322,129 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(envelopes[0].payload, b"\x92\xa5alpha\x01") self.assertEqual(envelopes[1].payload, b"\x92\xa4beta\x02") + def test_single_file_refusals_say_what_was_found_without_blaming_variants(self) -> None: + with self.assertRaises(ValueError) as unsupported: + self._pull_vector("unknown-header.msgpack") + self.assertIn("not one this runtime reads", str(unsupported.exception)) + self.assertNotIn("variant", str(unsupported.exception)) + with self.assertRaises(ValueError) as invalid: + self._pull_vector("extra-slot-full-payload.msgpack") + self.assertIn("not a valid store", str(invalid.exception)) + self.assertNotIn("variant", str(invalid.exception)) + + def test_single_file_only_a_missing_store_reads_as_empty(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(SingleFileMessagePackBackingStore(Path(tmp) / "store.cc").pull_all(), []) + + # An existing empty file is not a store: no CultCache writer leaves one. + empty = Path(tmp) / "empty.cc" + empty.write_bytes(b"") + with self.assertRaises(ValueError): + SingleFileMessagePackBackingStore(empty).pull_all() + self.assertEqual(empty.read_bytes(), b"") + + # An empty array is the legacy envelope array with no envelopes: no header, no format claimed. + legacy_empty = Path(tmp) / "legacy-empty.cc" + legacy_empty.write_bytes(b"\x90") + self.assertEqual(SingleFileMessagePackBackingStore(legacy_empty).pull_all(), []) + + def test_single_file_refusals_never_echo_a_value_from_the_store(self) -> None: + import msgpack # type: ignore + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + cases = [ + (["cultcache.store.v12", [], []], True), + (["cultcache.store.SECRET-HEADER", [], []], False), + (["cultcache.store.v12SECRET", [], []], False), + (["cultcache.store.v", [], []], False), + (["cultcache.store.v1", ["SECRET-CATALOG-TEXT"], []], None), + ] + for value, echoed in cases: + path.write_bytes(msgpack.packb(value, use_bin_type=True)) + with self.assertRaises(ValueError) as caught: + SingleFileMessagePackBackingStore(path).pull_all() + message = str(caught.exception) + if echoed: + self.assertIn(repr(value[0]), message) + else: + self.assertNotIn("SECRET", message) + if echoed is False: + self.assertNotIn(repr(value[0]), message) + self.assertIn(f"of {len(value[0])} bytes", message) + + def test_single_file_record_refusal_names_only_a_string_key(self) -> None: + import msgpack # type: ignore + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + for key, schema in ((["SECRET-IN-LIST"], "s"), (b"SECRET-BYTES", "s"), ("k", {"SECRET": 1})): + path.write_bytes(msgpack.packb( + ["cultcache.store.v1", [], [[key, schema, "t", b"\x90", "extra"]]], use_bin_type=True)) + with self.assertRaises(ValueError) as caught: + SingleFileMessagePackBackingStore(path).pull_all() + self.assertNotIn("SECRET", str(caught.exception)) + self.assertIn("not a valid store", str(caught.exception)) + + # What a write does through a symbolic link today. R3 decides whether a write resolves + # the link or refuses it; this pins the current behaviour so that change is deliberate. + @unittest.skipIf(os.name != "posix", "symbolic links need POSIX here") + def test_r3_decides_resolve_or_refuse_a_push_through_a_live_link_replaces_the_link(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + volume = Path(tmp) / "volume" + volume.mkdir() + target = volume / "store.cc" + envelope = CultCacheEnvelope(key="a", type="t", payload=b"\x90", stored_at="2026-09-30T00:00:00Z") + SingleFileMessagePackBackingStore(target).push(envelope) + before = target.read_bytes() + link = Path(tmp) / "store.cc" + link.symlink_to(target) + SingleFileMessagePackBackingStore(link).push(replace(envelope, key="b")) + self.assertFalse(link.is_symlink()) + self.assertTrue(link.is_file()) + self.assertEqual(target.read_bytes(), before) + + @unittest.skipIf(os.name != "posix", "symbolic links need POSIX here") + def test_a_dangling_symlink_at_a_store_path_is_an_os_error_and_nothing_is_written(self) -> None: + for store_type in (SingleFileMessagePackBackingStore, JsonLinesBackingStore): + with tempfile.TemporaryDirectory() as tmp: + volume = Path(tmp) / "volume" + volume.mkdir() + path = Path(tmp) / "store.cc" + path.symlink_to(volume / "store.cc") + store = store_type(path) + with self.assertRaises(FileNotFoundError): + store.pull_all() + with self.assertRaises(FileNotFoundError): + store.push(CultCacheEnvelope(key="k", type="t", payload=b"\x90", stored_at="2026-09-30T00:00:00Z")) + self.assertTrue(path.is_symlink()) + self.assertEqual(list(volume.iterdir()), []) + + @unittest.skipIf(os.name != "posix", "symlink loops and file-as-parent errors are POSIX errno cases") + def test_json_lines_store_that_cannot_be_reached_is_an_os_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(JsonLinesBackingStore(Path(tmp) / "missing.jsonl").pull_all(), []) + loop = Path(tmp) / "loop.jsonl" + loop.symlink_to(loop) + parent = Path(tmp) / "body" + parent.write_bytes(b"not a directory") + for path in (loop, parent / "store.jsonl"): + with self.assertRaises(OSError) as caught: + JsonLinesBackingStore(path).pull_all() + self.assertNotIsInstance(caught.exception, FileNotFoundError) + + @unittest.skipIf(os.name != "posix", "symlink loops and file-as-parent errors are POSIX errno cases") + def test_single_file_store_that_cannot_be_reached_is_an_os_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + loop = Path(tmp) / "loop.cc" + loop.symlink_to(loop) + parent = Path(tmp) / "body" + parent.write_bytes(b"not a directory") + for path in (loop, parent / "store.cc"): + with self.assertRaises(OSError) as caught: + SingleFileMessagePackBackingStore(path).pull_all() + self.assertNotIsInstance(caught.exception, FileNotFoundError) + # The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced. _C2A_VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a" diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 1ad01eb0..6841791f 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -293,6 +293,61 @@ writes. That protects ordinary multi-process access to the same file. It is still not a multi-master replication protocol, and it is not a substitute for a coordinator when higher-level write ordering matters. +A write that fails part way, for example on a full disk, removes its staging +file and leaves the store as it was. A read that fails tells you why through +the error chain: + +```rust +use cultcache_rs::{CacheBackingStore, CultCacheStoreUnreadable, CultCacheStoreUnreadableKind, SingleFileMessagePackBackingStore}; + +let store = SingleFileMessagePackBackingStore::new("cache.cc"); +if let Err(error) = store.pull_all() { + match error.downcast_ref::().map(|unreadable| unreadable.kind) { + // The bytes were read and are not a store: truncated, garbled, or invalid. + Some(CultCacheStoreUnreadableKind::Undecodable) => {} + // The header names a format this runtime does not read, such as a newer one. + Some(CultCacheStoreUnreadableKind::UnsupportedFormat) => {} + // The bytes could not be read at all: a std::io::Error is in the chain. + _ => {} + } +} +``` + +An I/O failure never carries `CultCacheStoreUnreadable`, and the chain under +`CultCacheStoreUnreadable` never holds a `std::io::Error`. + +Only a store file that does not exist (`std::io::ErrorKind::NotFound`) reads as +an empty store. Any other failure to reach it, such as a symlink loop, a file +where a parent directory should be, or a missing permission, is an I/O error. +A store file that exists and holds zero bytes is `Undecodable`: no CultCache +writer leaves one. A symbolic link whose target is gone is not a missing store: +reading it is an I/O error, so a read-then-write (`push`, `delete`, the exchanges) +fails before writing. Writes do not yet follow links: a write through a live link, +and a `push_all` through a dangling one, replace the link with a regular file and +leave its target alone. A refusal names no +value read from the file; it names a record only by its key and schema id, and +a header only when it has the shape `cultcache.store.v`. + +A write that fails says what it left on disk: + +```rust +use cultcache_rs::{CultCacheStoreWriteFailed, CultCacheStoreWriteFailedKind}; + +fn outcome(error: &anyhow::Error) { + match error.downcast_ref::().map(|failed| failed.kind) { + // The entries cannot be stored; retrying them fails the same way. + Some(CultCacheStoreWriteFailedKind::Rejected) => {} + // Staging, fsync or rename failed: the store file holds what it held before, + // and a retry may succeed once the fault (a full disk, say) clears. + Some(CultCacheStoreWriteFailedKind::NotReplaced) => {} + // The rename landed and the directory sync failed: the new snapshot is in + // place but may not survive a crash. + Some(CultCacheStoreWriteFailedKind::ReplacedNotDurable) => {} + _ => {} + } +} +``` + ## Near-Term Ergonomic Improvements 1. **Derive macro** diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index e8936fce..979bd6cf 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -280,8 +280,8 @@ struct PersistedRecord( ); impl<'de> serde::Deserialize<'de> for PersistedRecord { - /// Reads the four v1 slots and refuses a fifth, naming the record. A record with more - /// slots was written by a runtime that resolves document variants. + /// Reads the four v1 slots and refuses a fifth, naming the record. No runtime reads a + /// v1 store holding such a record, so the store is not a valid one. fn deserialize>(deserializer: D) -> std::result::Result { use serde::de::{Error, IgnoredAny, SeqAccess, Visitor}; @@ -301,7 +301,7 @@ impl<'de> serde::Deserialize<'de> for PersistedRecord { seq.next_element()?.ok_or_else(|| A::Error::invalid_length(3, &self))?; if seq.next_element::()?.is_some() { return Err(A::Error::custom(format!( - "CultCache record {key:?} (schema {schema_id:?}) has more than 4 slots; this runtime reads 4. The store needs a runtime that resolves document variants." + "{RECORD_REFUSAL}{key:?} (schema {schema_id:?}) has more than the 4 slots of a {STORE_FORMAT_V1} record, so this is not a valid store" ))); } Ok(PersistedRecord(key, schema_id, stored_at, payload.into_vec())) @@ -319,6 +319,61 @@ const STORE_FORMAT_V1: &str = "cultcache.store.v1"; /// found under it; it does not decide when a document carries ids. const STORE_FORMAT_ELEMENT_IDS: &str = "cultcache.store.v3"; +/// Opens every refusal CultCache raises about one persisted record. The decoder's own +/// messages can quote the bytes; these name only the record's key and schema id. +const RECORD_REFUSAL: &str = "CultCache record "; + +/// A store header as a refusal may show it: echoed when it has the shape +/// `cultcache.store.v`, and otherwise described only by its length, since the +/// bytes are the store's and an error text must not carry them. +fn describe_header(header: &str) -> String { + let known_shape = header + .strip_prefix("cultcache.store.v") + .is_some_and(|version| !version.is_empty() && version.bytes().all(|byte| byte.is_ascii_digit())); + if known_shape { + format!("{header:?}") + } else { + format!("an unrecognised cultcache.store.* header of {} bytes", header.len()) + } +} + +/// What a failed decode tells the operator: CultCache's own context and record +/// refusals, and for a decoder failure only its kind, never a value from the bytes. +fn describe_undecodable(error: &anyhow::Error) -> String { + let mut parts = Vec::new(); + for cause in error.chain() { + if let Some(decode) = cause.downcast_ref::() { + parts.push(describe_decode_error(decode)); + break; + } + parts.push(cause.to_string()); + } + parts.join(": ") +} + +/// A decoder failure as an error that keeps only its kind. Every decode of stored bytes +/// goes through this or [`describe_undecodable`], so no error text quotes a stored value. +fn opaque_decode_error(error: rmp_serde::decode::Error) -> anyhow::Error { + anyhow!(describe_decode_error(&error)) +} + +fn describe_decode_error(error: &rmp_serde::decode::Error) -> String { + use rmp_serde::decode::Error; + match error { + Error::InvalidMarkerRead(_) | Error::InvalidDataRead(_) => "the bytes end before the store does", + Error::TypeMismatch(_) => "a value has a MessagePack type the store format does not allow", + Error::OutOfRange => "a number is out of range for its slot", + Error::LengthMismatch(_) => "an array has a length the store format does not allow", + Error::Utf8Error(_) => "a string is not valid UTF-8", + Error::DepthLimitExceeded => "values are nested deeper than the decoder allows", + Error::Syntax(message) | Error::Uncategorized(message) if message.starts_with(RECORD_REFUSAL) => { + return message.clone(); + } + Error::Syntax(_) | Error::Uncategorized(_) => "a value does not fit the store format", + } + .to_string() +} + /// The store header string, `None` when the first slot is not a string (a legacy envelope array candidate), and an error /// for bytes that are not exactly one complete MessagePack array: a truncated store or bytes after the array are refused, /// never taken for a legacy file. One pass reads the first slot and walks the rest to prove the file whole. @@ -349,46 +404,14 @@ fn store_header(bytes: &[u8]) -> Result> { _ => None, })) } - } - deserializer.deserialize_seq(HeaderVisitor) - } - } let mut cursor = std::io::Cursor::new(bytes); let header =
::deserialize(&mut rmp_serde::Deserializer::new(&mut cursor)) - .map_err(|error| anyhow!("the store is not one complete MessagePack array: {error}"))?; + .map_err(|error| opaque_decode_error(error).context("the store is not one complete MessagePack array"))?; let after = bytes.len() - cursor.position() as usize; ensure!(after == 0, "the store has {after} bytes after its MessagePack array"); Ok(header.0) } -/// A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra -/// slot), a header or record it does not know, or a body it cannot decode. Open, every rewrite and the read-only snapshot -/// refuse a file with this error, and a refused file is left as it was. The cause is the error's source. Test for it with -/// `error.downcast_ref::()`. -#[derive(Debug)] -pub struct StoreUnreadableError { - path: PathBuf, - source: anyhow::Error, -} - -impl StoreUnreadableError { - pub fn path(&self) -> &Path { - &self.path - } -} - -impl std::fmt::Display for StoreUnreadableError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "failed to decode MessagePack {}", self.path.display()) - } -} - -impl std::error::Error for StoreUnreadableError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.source.chain().next() - } -} - /// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried /// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key. #[derive(Debug)] @@ -410,6 +433,91 @@ impl std::fmt::Display for SchemaConflictError { impl std::error::Error for SchemaConflictError {} +/// A single-file store whose bytes were read and not accepted. It sits in the error +/// chain of every such read, so a consumer tells it apart with +/// `error.downcast_ref::()`. A failure to read the bytes +/// at all stays a [`std::io::Error`] in the chain and never carries this type; the +/// chain under this type never holds an `io::Error`, so the two never overlap. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct CultCacheStoreUnreadable { + pub path: PathBuf, + pub kind: CultCacheStoreUnreadableKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum CultCacheStoreUnreadableKind { + /// The bytes are not a store this runtime's format accepts: truncated, garbled, + /// or a record the format does not allow. + Undecodable, + /// The header names a store format this runtime does not read, such as one + /// written by a newer runtime. The rest of the file was not judged. + UnsupportedFormat, +} + +impl std::fmt::Display for CultCacheStoreUnreadable { + fn fmt(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.kind { + CultCacheStoreUnreadableKind::Undecodable => { + write!(formatter, "CultCache store {} is not decodable", self.path.display()) + } + CultCacheStoreUnreadableKind::UnsupportedFormat => write!( + formatter, + "CultCache store {} is in a format this runtime does not read", + self.path.display() + ), + } + } +} + +impl std::error::Error for CultCacheStoreUnreadable {} + +/// A write to a single-file store that failed, and what it left on disk. It sits in the +/// error chain of every such failure, above the cause, so a consumer tells the outcomes +/// apart with `error.downcast_ref::()`. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct CultCacheStoreWriteFailed { + pub path: PathBuf, + pub kind: CultCacheStoreWriteFailedKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum CultCacheStoreWriteFailedKind { + /// The entries cannot be stored as a snapshot, for example two document types + /// claim one schema id. Nothing was written, and retrying the same entries fails + /// the same way: this is a refusal, not an I/O failure. + Rejected, + /// The write failed before the store file was replaced (creating, writing or + /// syncing the staging file, or renaming it): the file holds what it held before. + NotReplaced, + /// The store file was replaced, then syncing its directory failed: the new + /// snapshot is in place but may not survive a crash. + ReplacedNotDurable, +} + +impl std::fmt::Display for CultCacheStoreWriteFailed { + fn fmt(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.kind { + CultCacheStoreWriteFailedKind::Rejected => { + write!(formatter, "CultCache store {} refused a snapshot it cannot store", self.path.display()) + } + CultCacheStoreWriteFailedKind::NotReplaced => { + write!(formatter, "CultCache store {} was not replaced", self.path.display()) + } + CultCacheStoreWriteFailedKind::ReplacedNotDurable => write!( + formatter, + "CultCache store {} was replaced, but the replacement is not yet durable", + self.path.display() + ), + } + } +} + +impl std::error::Error for CultCacheStoreWriteFailed {} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PushAllOptions { pub soft: bool, @@ -486,10 +594,12 @@ impl SingleFileMessagePackBackingStore { /// either file. Writers using the sibling lock cannot replace the snapshot /// until `action` returns. /// - /// Releasing the lock is RAII cleanup. A release failure therefore cannot - /// rewrite the completed action's result; consumers that need to observe - /// that cleanup failure can use - /// [`Self::with_read_only_shared_snapshot_and_unlock_diagnostic`]. + /// Releasing the lock is RAII cleanup, under the same policy as every lock in + /// this crate (see `release_lock`): a release failure cannot rewrite the + /// completed action's result. This path alone also offers an observer for that + /// failure, [`Self::with_read_only_shared_snapshot_and_unlock_diagnostic`], + /// because a provider crossing may need to report it; the locked read and write + /// paths discard it. pub fn with_read_only_shared_snapshot( &self, action: impl FnOnce(Vec) -> Result, @@ -785,21 +895,43 @@ impl SingleFileMessagePackBackingStore { } /// The one reader of the store file: open, every rewrite and the read-only snapshot ask it, so they agree on which - /// files are stores. A file that is gone or zero bytes is an empty v1 store. Anything else must decode completely, as - /// exactly one store or one legacy envelope array; a file this runtime cannot read is refused with - /// [`StoreUnreadableError`], so a rewrite never overwrites what it cannot see. + /// files are stores. Only a file that is not there is an empty store. A file that is there and empty is not a store + /// (no writer leaves one), and any other failure to reach the file, such as a symlink loop, a file where a directory + /// should be, or no permission, stays an io::Error: reading it as empty would let a writer replace a store it never + /// saw. A dangling symbolic link is something at the path, not nothing, for the same reason. Anything else must decode + /// completely, as exactly one store or one legacy envelope array; bytes this runtime cannot read are refused with + /// [`CultCacheStoreUnreadable`], so a rewrite never overwrites what it cannot see. fn read_store_unlocked(&self) -> Result { let bytes = match fs::read(&self.path) { Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(DiskStore::empty()), - Err(error) => return Err(error).with_context(|| format!("failed to read {}", self.path.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound && is_absent(&self.path)? => { + return Ok(DiskStore::empty()); + } + Err(error) => { + return Err(anyhow::Error::new(error) + .context(format!("failed to read {}", self.path.display()))); + } }; - if bytes.is_empty() { - return Ok(DiskStore::empty()); + let unreadable = |kind, detail: anyhow::Error| { + // The detail is rebuilt as text that names no value read from the bytes. The + // decoder reports a short read as an io::Error, and dropping it keeps an + // io::Error in the chain meaning only that the file itself could not be read. + anyhow!(describe_undecodable(&detail)).context(CultCacheStoreUnreadable { + path: self.path.clone(), + kind, + }) + }; + let header = store_header(&bytes).map_err(|error| unreadable(CultCacheStoreUnreadableKind::Undecodable, error))?; + match header { + Some(header) if header.starts_with("cultcache.store.") && readable_store_format(&header).is_none() => Err(unreadable( + CultCacheStoreUnreadableKind::UnsupportedFormat, + anyhow!( + "its header is {}; this runtime reads {STORE_FORMAT_V1:?} and {STORE_FORMAT_ELEMENT_IDS:?} only", + describe_header(&header) + ), + )), + _ => decode_store_file(&bytes).map_err(|error| unreadable(CultCacheStoreUnreadableKind::Undecodable, error)), } - decode_store_file(&bytes).map_err(|source| { - anyhow::Error::new(StoreUnreadableError { path: self.path.clone(), source }) - }) } /// Reads one filesystem snapshot without creating or opening the sibling @@ -815,34 +947,44 @@ impl SingleFileMessagePackBackingStore { } fn write_all_unlocked(&self, entries: &[CultCacheEnvelope]) -> Result<()> { + let failed = |kind| CultCacheStoreWriteFailed { + path: self.path.clone(), + kind, + }; + // The header a rewrite writes is the one the file on disk carries: a file marked for element ids stays marked (the + // payloads this store passes through may hold ids), one that is gone or legacy is written v1. The file is read by + // the same reader as an open, so a store this runtime cannot read completely, a variant included, is refused + // and never overwritten. + let disk = self.read_store_unlocked()?; + let bytes = encode_store_snapshot(entries, &disk) + .and_then(|snapshot| rmp_serde::to_vec(&snapshot).context("failed to encode MessagePack")) + .map_err(|error| error.context(failed(CultCacheStoreWriteFailedKind::Rejected)))?; + self.replace_unlocked(&bytes) + .map_err(|error| error.context(failed(CultCacheStoreWriteFailedKind::NotReplaced)))?; + sync_parent_directory(&self.path) + .map_err(|error| error.context(failed(CultCacheStoreWriteFailedKind::ReplacedNotDurable))) + } + + /// Everything up to and including the rename over the store file. On failure the + /// store file is as it was. + fn replace_unlocked(&self, bytes: &[u8]) -> Result<()> { if let Some(parent) = self.path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; } remove_abandoned_staging_files(&self.path)?; - // The header a rewrite writes is the one the file on disk carries: a file marked for element ids stays marked (the - // payloads this store passes through may hold ids), one that is gone, empty or legacy is written v1. The file is read - // by the same reader as an open, so a store this runtime cannot read completely, a variant included, is refused - // and never overwritten. - let disk = self.read_store_unlocked()?; - let bytes = rmp_serde::to_vec(&encode_store_snapshot(entries, &disk)?) - .context("failed to encode MessagePack")?; let tmp_path = temporary_path_for(&self.path); - let mut staged = OpenOptions::new() + let staged = OpenOptions::new() .create_new(true) .write(true) .open(&tmp_path) .with_context(|| format!("failed to create {}", tmp_path.display()))?; - use std::io::Write; - staged - .write_all(&bytes) - .with_context(|| format!("failed to write {}", tmp_path.display()))?; - staged - .sync_all() - .with_context(|| format!("failed to sync {}", tmp_path.display()))?; - drop(staged); - replace_file_atomically(&tmp_path, &self.path)?; - sync_parent_directory(&self.path)?; + // From here the staging file is this write's to remove: a write that fails + // for lack of space must not keep holding the space it failed on. + if let Err(error) = stage_and_replace(staged, &tmp_path, bytes, &self.path) { + let _ = fs::remove_file(&tmp_path); + return Err(error); + } Ok(()) } @@ -850,20 +992,14 @@ impl SingleFileMessagePackBackingStore { let lock = self.open_lock_file()?; fs2::FileExt::lock_shared(&lock) .with_context(|| format!("failed to lock {}", self.lock_path().display()))?; - let result = action(); - fs2::FileExt::unlock(&lock) - .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; - result + release_lock(lock, action()) } fn with_exclusive_lock(&self, action: impl FnOnce() -> Result) -> Result { let lock = self.open_lock_file()?; fs2::FileExt::lock_exclusive(&lock) .with_context(|| format!("failed to lock {}", self.lock_path().display()))?; - let result = action(); - fs2::FileExt::unlock(&lock) - .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; - result + release_lock(lock, action()) } /// Replaces the named entries only if each still matches its expectation. @@ -980,10 +1116,7 @@ impl SingleFileMessagePackBackingStore { .with_context(|| format!("failed to lock {}", self.lock_path().display())); } } - let result = action(); - fs2::FileExt::unlock(&lock) - .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; - result.map(Some) + release_lock(lock, action()).map(Some) } fn open_lock_file(&self) -> Result { @@ -1068,7 +1201,7 @@ fn unique_batch_ids( impl CacheBackingStore for SingleFileMessagePackBackingStore { fn pull_all(&self) -> Result> { - if !self.path.exists() && !self.lock_path().exists() { + if is_absent(&self.path)? && is_absent(&self.lock_path())? { return Ok(Vec::new()); } self.with_shared_lock(|| self.read_all_unlocked()) @@ -1173,10 +1306,7 @@ impl RedbMessagePackBackingStore { .with_context(|| format!("failed to open {}", lock_path.display()))?; fs2::FileExt::lock_exclusive(&lock) .with_context(|| format!("failed to lock {}", lock_path.display()))?; - let result = self.open_database().and_then(|database| action(&database)); - fs2::FileExt::unlock(&lock) - .with_context(|| format!("failed to unlock {}", lock_path.display()))?; - result + release_lock(lock, self.open_database().and_then(|database| action(&database))) } fn lock_path(&self) -> PathBuf { @@ -1956,7 +2086,9 @@ fn read_redb_entry( table .get(key.as_slice())? .map(|value| { - rmp_serde::from_slice(value.value()).context("failed to decode redb CultCache envelope") + rmp_serde::from_slice(value.value()) + .map_err(opaque_decode_error) + .context("failed to decode redb CultCache envelope") }) .transpose() } @@ -1978,7 +2110,9 @@ fn read_all_redb( .iter()? .map(|row| { let (_, value) = row?; - rmp_serde::from_slice(value.value()).context("failed to decode redb CultCache envelope") + rmp_serde::from_slice(value.value()) + .map_err(opaque_decode_error) + .context("failed to decode redb CultCache envelope") }) .collect::>>()?; entries.sort_by_key(entry_id); @@ -2210,7 +2344,7 @@ impl CultCache { let Some(entry) = self.entries.get(&entry_id_parts(T::TYPE, key)) else { return Ok(None); }; - let payload = rmp_serde::from_slice(&entry.payload).with_context(|| { + let payload = rmp_serde::from_slice(&entry.payload).map_err(opaque_decode_error).with_context(|| { format!( "failed to decode CultCache entry {:?} at key {:?} as {}", T::TYPE, @@ -2243,7 +2377,7 @@ impl CultCache { if entry.r#type != T::TYPE { continue; } - values.push(rmp_serde::from_slice(&entry.payload).with_context(|| { + values.push(rmp_serde::from_slice(&entry.payload).map_err(opaque_decode_error).with_context(|| { format!( "failed to decode CultCache entry {:?} at key {:?} as {}", T::TYPE, @@ -2264,7 +2398,7 @@ impl CultCache { } values.push(( entry.key.clone(), - rmp_serde::from_slice(&entry.payload).with_context(|| { + rmp_serde::from_slice(&entry.payload).map_err(opaque_decode_error).with_context(|| { format!( "failed to decode CultCache entry {:?} at key {:?} as {}", T::TYPE, @@ -2346,7 +2480,7 @@ impl CultCache { key: String, payload: Vec, ) -> Result<(CultCacheEnvelope, T)> { - let parsed: T = rmp_serde::from_slice(&payload).with_context(|| { + let parsed: T = rmp_serde::from_slice(&payload).map_err(opaque_decode_error).with_context(|| { format!( "failed to validate CultCache entry {:?} at key {:?} as {}", T::TYPE, @@ -2482,7 +2616,7 @@ impl CultCache { )); } - rmp_serde::from_slice(&entry.payload).with_context(|| { + rmp_serde::from_slice(&entry.payload).map_err(opaque_decode_error).with_context(|| { format!( "failed to validate CultCache envelope {:?} at key {:?} as {}", T::TYPE, @@ -2707,13 +2841,11 @@ fn described(schema_id: &str, document_type: String) -> PersistedSchemaCatalogEn ) } -fn readable_store_format(header: &str) -> Result<&'static str> { +fn readable_store_format(header: &str) -> Option<&'static str> { match header { - STORE_FORMAT_V1 => Ok(STORE_FORMAT_V1), - STORE_FORMAT_ELEMENT_IDS => Ok(STORE_FORMAT_ELEMENT_IDS), - _ => Err(anyhow!( - "CultCache store format {header:?} is not readable; this runtime reads \"cultcache.store.v1\" and \"cultcache.store.v3\" only. The store needs a runtime that resolves document variants." - )), + STORE_FORMAT_V1 => Some(STORE_FORMAT_V1), + STORE_FORMAT_ELEMENT_IDS => Some(STORE_FORMAT_ELEMENT_IDS), + _ => None, } } @@ -2722,7 +2854,8 @@ fn readable_store_format(header: &str) -> Result<&'static str> { fn decode_store_file(bytes: &[u8]) -> Result { match store_header(bytes)? { Some(header) => { - let format = readable_store_format(&header)?; + let format = readable_store_format(&header) + .ok_or_else(|| anyhow!("the first slot is not a store header this runtime reads"))?; let (envelopes, catalog) = decode_store_snapshot(bytes)?; Ok(DiskStore { format, envelopes, catalog }) } @@ -2739,7 +2872,7 @@ struct DiskStore { } impl DiskStore { - /// A file that is gone or zero bytes: an empty v1 store. + /// A file that is gone: an empty v1 store. fn empty() -> Self { Self { format: STORE_FORMAT_V1, envelopes: Vec::new(), catalog: Vec::new() } } @@ -2751,6 +2884,7 @@ fn decode_legacy_envelopes(bytes: &[u8]) -> Result> { rmp_serde::from_slice(bytes).map_err(anyhow::Error::from) } +/// Decodes a store whose header the caller has already read as a readable one: its envelopes and its catalog. fn decode_store_snapshot(bytes: &[u8]) -> Result<(Vec, Vec)> { let snapshot: PersistedStoreSnapshot = rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; @@ -2824,6 +2958,47 @@ fn temporary_path_for(path: &Path) -> PathBuf { path.with_file_name(file_name) } +/// Writes and syncs the staged snapshot, closes it, and renames it over `destination`. +fn stage_and_replace(mut staged: File, staged_path: &Path, bytes: &[u8], destination: &Path) -> Result<()> { + use std::io::Write; + staged + .write_all(bytes) + .with_context(|| format!("failed to write {}", staged_path.display()))?; + injected_write_fault(WriteStep::StagingSync) + .and_then(|()| staged.sync_all()) + .with_context(|| format!("failed to sync {}", staged_path.display()))?; + drop(staged); + replace_file_atomically(staged_path, destination) +} + +/// Releases a lock taken around `result`'s action and returns that result. +/// +/// The policy for every lock in this crate: an unlock failure never replaces the action's +/// result, so a write that landed is reported as landed. The failure is discarded here +/// because these paths have no observer; the read-only snapshot, whose API offers one, +/// hands it to [`SingleFileMessagePackBackingStore::with_read_only_shared_snapshot_and_unlock_diagnostic`]. +/// +/// Dropping `lock` closes this descriptor, which releases the lock only when no other +/// descriptor shares its open file description. A child forked without exec holds such a +/// duplicate, and the lock then stays held until the child closes it or exits. +fn release_lock(lock: File, result: Result) -> Result { + let _unlock_failure_released_by_close = + injected_write_fault(WriteStep::Unlock).and_then(|()| fs2::FileExt::unlock(&lock)); + drop(lock); + result +} + +/// True only when nothing is at `path`; a symbolic link, dangling or not, is something. +/// A path that cannot be inspected (a file where a directory should be, no permission) +/// is an error, never absent. +fn is_absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Err(error) => Err(anyhow::Error::new(error).context(format!("failed to inspect {}", path.display()))), + } +} + fn remove_abandoned_staging_files(destination: &Path) -> Result<()> { let Some(parent) = destination.parent() else { return Ok(()); @@ -2967,11 +3142,41 @@ fn sync_parent_directory(path: &Path) -> Result<()> { let Some(parent) = path.parent() else { return Ok(()); }; - File::open(parent) + injected_write_fault(WriteStep::DirectorySync) + .and_then(|()| File::open(parent)) .and_then(|directory| directory.sync_all()) .with_context(|| format!("failed to sync {}", parent.display())) } +/// The steps no test can make fail for real: fsync and unlock do not fail on a healthy +/// filesystem. Tests arm one step on their own thread; builds outside tests never fail here. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[allow(dead_code)] +enum WriteStep { + StagingSync, + DirectorySync, + Unlock, +} + +#[cfg(test)] +thread_local! { + static INJECTED_WRITE_FAULT: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +#[cfg(test)] +fn injected_write_fault(step: WriteStep) -> std::io::Result<()> { + if INJECTED_WRITE_FAULT.with(|fault| fault.get()) == Some(step) { + return Err(std::io::Error::other("injected write fault")); + } + Ok(()) +} + +#[cfg(not(test))] +#[inline(always)] +fn injected_write_fault(_step: WriteStep) -> std::io::Result<()> { + Ok(()) +} + #[cfg(windows)] fn sync_parent_directory(_path: &Path) -> Result<()> { // MoveFileExW with MOVEFILE_WRITE_THROUGH owns publication durability. @@ -3090,6 +3295,8 @@ mod tests { store.push(&released)?; let writer_path = store_path.clone(); + let gate_path = store_path.clone(); + let gate_released = released.clone(); let (started_tx, started_rx) = mpsc::channel(); let (finished_tx, finished_rx) = mpsc::channel(); store.with_read_only_shared_snapshot(|captured| { @@ -3104,15 +3311,21 @@ mod tests { writer.push(&engaged).unwrap(); finished_tx.send(()).unwrap(); }); - started_rx.recv_timeout(Duration::from_secs(1))?; + started_rx.recv_timeout(Duration::from_secs(30))?; assert!( finished_rx .recv_timeout(Duration::from_millis(100)) .is_err() ); + // The writer is still waiting on the gate: the file on disk is untouched. + assert_eq!( + SingleFileMessagePackBackingStore::new(&gate_path).pull_all_read_only_snapshot()?, + vec![gate_released.clone()] + ); Ok(()) })?; - finished_rx.recv_timeout(Duration::from_secs(1))?; + // The writer fsyncs twice once the gate opens; under IO pressure that is slow. + finished_rx.recv_timeout(Duration::from_secs(60))?; assert_eq!(store.pull_all()?[0].payload, b"engaged"); Ok(()) } @@ -4505,7 +4718,7 @@ mod tests { cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; // The payload opens with a schema version, and the catalog is the store's own description: no recovery. let error = cache.pull_all_backing_stores().unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert!(error.downcast_ref::().is_some(), "{error:#}"); Ok(()) } @@ -4866,7 +5079,7 @@ mod tests { // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by open, // push_all and push alike. The bytes and every runtime's verdict are tests/vectors/document-variants-c2a/readability. - // A refusal is a StoreUnreadableError, and a replaced file carries the header its old content decides. + // A refusal is a CultCacheStoreUnreadable, and a replaced file carries the header its old content decides. #[test] fn a_file_is_replaced_exactly_when_it_reads() -> Result<()> { let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/readability"); @@ -4895,11 +5108,13 @@ mod tests { assert_eq!(outcome.is_ok(), reads, "{what}: {outcome:?}"); if !reads { let error = outcome.unwrap_err(); - let typed = error.downcast_ref::().unwrap_or_else(|| panic!("{what}: {error:#}")); - let cause = std::error::Error::source(typed).unwrap_or_else(|| panic!("{what}: the cause is the source")).to_string(); - // The message is the error's own; the cause is printed once, by the chain. - assert!(!typed.to_string().contains(&cause), "{what}: the message repeats the cause: {typed}"); - assert_eq!(format!("{error:#}").matches(&cause).count(), 1, "{what}: {error:#}"); + let expected = if vector == "header-v9.bin" { + CultCacheStoreUnreadableKind::UnsupportedFormat + } else { + CultCacheStoreUnreadableKind::Undecodable + }; + assert_eq!(unreadable_kind(&error), Some(expected), "{what}: {error:#}"); + assert!(io_error_in(&error).is_none(), "{what}: a refusal is not an I/O failure: {error:#}"); assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read"); } else if operation != "open" { assert_eq!(store_header(&std::fs::read(&path)?)?.as_deref(), Some(header), "{what}"); @@ -5024,24 +5239,276 @@ mod tests { Ok(()) } - // Only a file that is not there is an empty store; any other failure to read it is an error, so a write never replaces a - // file it could not read. + // Store read and write faults: what a consumer can tell apart, and what a failed + // write leaves on disk. + + fn unreadable_kind(error: &anyhow::Error) -> Option { + error + .downcast_ref::() + .map(|unreadable| unreadable.kind) + } + + fn io_error_in(error: &anyhow::Error) -> Option<&std::io::Error> { + error.chain().find_map(|cause| cause.downcast_ref::()) + } + + /// A store of three records written by this runtime, returned with its bytes. + fn written_store(path: &Path) -> Result<(SingleFileMessagePackBackingStore, Vec)> { + let store = SingleFileMessagePackBackingStore::new(path); + let records = vec![ + snapshot_envelope("alpha", &[1; 64]), + snapshot_envelope("beta", &[2; 64]), + snapshot_envelope("gamma", &[3; 64]), + ]; + assert!(store.compare_exchange_snapshot(&[], &records)?); + let bytes = fs::read(path)?; + Ok((store, bytes)) + } + #[test] - fn a_store_path_that_cannot_be_read_is_an_error_not_an_empty_store() -> Result<()> { + fn a_truncated_store_is_undecodable_on_every_read_and_is_left_as_it_was() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (store, bytes) = written_store(&path)?; + let truncated = &bytes[..bytes.len() / 2]; + fs::write(&path, truncated)?; + + let failures = [ + store.pull_all().unwrap_err(), + store.pull_all_read_only_snapshot().unwrap_err(), + store + .compare_exchange_snapshot(&[], &[snapshot_envelope("delta", b"4")]) + .unwrap_err(), + ]; + for error in &failures { + assert_eq!(unreadable_kind(error), Some(CultCacheStoreUnreadableKind::Undecodable), "{error:#}"); + assert_eq!(error.downcast_ref::().unwrap().path, path); + assert!(error.to_string().contains(&path.display().to_string()), "{error}"); + assert!(io_error_in(error).is_none(), "{error:#}"); + } + assert_eq!(fs::read(&path)?, truncated); + Ok(()) + } + + #[test] + fn bytes_that_are_not_a_store_are_undecodable() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + // The last opens like a store, with a string where the header goes, but only a + // `cultcache.store.*` header claims a format this runtime might not read. + for bytes in [ + &b"not a store at all"[..], + &[0x92, 0xa3, b'a', b'b'][..], + &[0xc1][..], + &[0x92, 0xa2, b'a', b'b', 0x90][..], + &[0x93, 0xb0, b'c', b'u', b'l', b't', b'c', b'a', b'c', b'h', b'e', b'.', b's', b't', b'o', b'r', b'e', b'X', 0x90, 0x90][..], + ] { + fs::write(&path, bytes)?; + let error = store.pull_all().unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::Undecodable), "{bytes:?}: {error:#}"); + } + Ok(()) + } + + #[test] + fn a_v1_store_holding_a_record_v1_does_not_allow_is_undecodable() { + let error = pull_vector("extra-slot-full-payload.msgpack").unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::Undecodable), "{error:#}"); + } + + #[test] + fn a_store_in_a_format_this_runtime_does_not_read_is_unsupported_even_when_cut_short() -> Result<()> { + for name in ["unknown-header.msgpack", "variant-v2.msgpack", "../document-variants-c1/variant-store.msgpack"] { + let error = pull_vector(name).unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{name}: {error:#}"); + } + + // The header is all this runtime judges: a newer store cut short is still a + // newer store, not a damaged one this runtime could replace. let temp = tempfile::tempdir()?; let path = temp.path().join("store.msgpack"); - std::fs::create_dir(&path)?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - assert!(store.pull_all().is_err(), "a directory is not an empty store"); - let own = CultCacheEnvelope { - key: "b".into(), - r#type: "tests.other".into(), - payload: b"two".to_vec(), - stored_at: "2026-09-30T00:00:00Z".into(), - schema_id: Some("tests.other".into()), - }; - assert!(store.push(&own).is_err()); - assert!(path.is_dir(), "the path is left as it was"); + let vector = fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c0/unknown-header.msgpack"), + )?; + let truncated = &vector[..vector.len() / 2]; + fs::write(&path, truncated)?; + let store = SingleFileMessagePackBackingStore::new(&path); + for error in [store.pull_all().unwrap_err(), store.pull_all_read_only_snapshot().unwrap_err()] { + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{error:#}"); + assert!(format!("{error:#}").contains("cultcache.store.v9"), "{error:#}"); + assert!(io_error_in(&error).is_none(), "{error:#}"); + } + assert_eq!(fs::read(&path)?, truncated); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_an_io_error_and_never_unreadable() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + fs::create_dir(&path)?; + let store = SingleFileMessagePackBackingStore::new(&path); + for error in [store.pull_all().unwrap_err(), store.pull_all_read_only_snapshot().unwrap_err()] { + assert!(io_error_in(&error).is_some(), "{error:#}"); + assert_eq!(unreadable_kind(&error), None, "{error:#}"); + } + Ok(()) + } + + /// Runs one write in a child process whose file-size limit is far below the + /// snapshot, with SIGXFSZ ignored so the write fails with EFBIG instead of + /// killing the child: a real mid-write failure after the staging file exists, + /// reached without permissions (the verification container runs as root). + #[cfg(unix)] + #[test] + fn a_write_that_fails_part_way_leaves_no_staging_file_and_keeps_the_store() -> Result<()> { + const CHILD_STORE: &str = "CULTCACHE_RS_FILE_SIZE_LIMITED_STORE"; + if let Ok(path) = std::env::var(CHILD_STORE) { + let mut store = SingleFileMessagePackBackingStore::new(path); + let error = store + .push(&snapshot_envelope("large", &vec![7; 1 << 20])) + .unwrap_err(); + let io = io_error_in(&error).expect("a size-limited write fails with an io::Error"); + assert_eq!(io.kind(), std::io::ErrorKind::FileTooLarge, "{error:#}"); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert_eq!(unreadable_kind(&error), None, "{error:#}"); + return Ok(()); + } + + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (store, before) = written_store(&path)?; + let child = std::process::Command::new("sh") + .arg("-c") + .arg("trap '' XFSZ; ulimit -f 64; exec \"$0\" --exact --test-threads=1 --quiet tests::a_write_that_fails_part_way_leaves_no_staging_file_and_keeps_the_store") + .arg(std::env::current_exe()?) + .env(CHILD_STORE, &path) + .output()?; + assert!( + child.status.success(), + "the size-limited child did not fail its write as expected: {} +{}{}", + child.status, + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + + let staging: Vec<_> = fs::read_dir(temp.path())? + .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) + .collect::>>()? + .into_iter() + .filter(|name| name.starts_with("store.cc.") && name.ends_with(".tmp")) + .collect(); + assert_eq!(staging, Vec::::new()); + assert_eq!(fs::read(&path)?, before); + assert_eq!(store.pull_all()?.len(), 3); + Ok(()) + } + + /// Every read of a single-file store: the locked pull, the lock-free provider + /// snapshot, and the exchange a writer reads through. + fn every_read(store: &SingleFileMessagePackBackingStore) -> [Result>; 3] { + [ + store.pull_all(), + store.pull_all_read_only_snapshot(), + store + .compare_exchange_snapshot(&[], &[snapshot_envelope("delta", b"4")]) + .map(|_| Vec::new()), + ] + } + + fn assert_io_failure_on_every_read(store: &SingleFileMessagePackBackingStore) { + for result in every_read(store) { + let error = result.expect_err("a store that cannot be reached must not read as empty"); + let io = io_error_in(&error).expect("the failure to reach the store is an io::Error"); + assert_ne!(io.kind(), std::io::ErrorKind::NotFound, "{error:#}"); + assert_eq!(unreadable_kind(&error), None, "{error:#}"); + } + } + + #[test] + fn only_a_missing_store_reads_as_empty() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + assert!(store.pull_all()?.is_empty()); + assert!(store.pull_all_read_only_snapshot()?.is_empty()); + assert!(store.compare_exchange_snapshot(&[], &[snapshot_envelope("delta", b"4")])?); + + // A store whose lock file is gone is still a store, not an absent one. + fs::remove_file(store.lock_path())?; + assert_eq!(store.pull_all()?, vec![snapshot_envelope("delta", b"4")]); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_store_behind_a_symlink_loop_is_an_io_error_on_every_read() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + std::os::unix::fs::symlink(&path, &path)?; + assert_io_failure_on_every_read(&SingleFileMessagePackBackingStore::new(&path)); + assert!(fs::symlink_metadata(&path)?.file_type().is_symlink()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_store_under_a_file_instead_of_a_directory_is_an_io_error_on_every_read() -> Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("body"); + fs::write(&parent, b"not a directory")?; + assert_io_failure_on_every_read(&SingleFileMessagePackBackingStore::new(parent.join("store.cc"))); + assert_eq!(fs::read(&parent)?, b"not a directory"); + Ok(()) + } + + #[test] + fn an_existing_empty_file_is_undecodable_and_is_left_as_it_was() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + fs::write(&path, b"")?; + let store = SingleFileMessagePackBackingStore::new(&path); + for result in every_read(&store) { + let error = result.expect_err("no CultCache writer leaves an empty file, so it is not a store"); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::Undecodable), "{error:#}"); + assert!(io_error_in(&error).is_none(), "{error:#}"); + } + assert_eq!(fs::read(&path)?, b""); + + // An empty array is the legacy envelope array with no envelopes: it has no + // header, so it claims no format, and it is an empty store. + fs::write(&path, [0x90])?; + assert!(store.pull_all()?.is_empty()); + assert!(store.pull_all_read_only_snapshot()?.is_empty()); + Ok(()) + } + + #[test] + fn a_newer_store_is_named_once_its_header_string_is_complete() -> Result<()> { + let vector = fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c0/unknown-header.msgpack"), + )?; + // One byte of array header, one of string header, then the 18-byte header string. + let header = "cultcache.store.v9"; + let header_end = 2 + header.len(); + assert_eq!(header_end, 20); + assert_eq!(&vector[2..header_end], header.as_bytes()); + + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + for length in 1..=vector.len() { + fs::write(&path, &vector[..length])?; + let error = store.pull_all().unwrap_err(); + let expected = if length < header_end { + CultCacheStoreUnreadableKind::Undecodable + } else { + CultCacheStoreUnreadableKind::UnsupportedFormat + }; + assert_eq!(unreadable_kind(&error), Some(expected), "cut to {length}: {error:#}"); + } Ok(()) } @@ -5173,13 +5640,91 @@ mod tests { _ => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()), }; let error = outcome.expect_err(&format!("{vector} {operation}")); - assert!(error.downcast_ref::().is_some(), "{vector} {operation}: {error:#}"); + assert!(unreadable_kind(&error).is_some(), "{vector} {operation}: {error:#}"); assert_eq!(std::fs::read(&path)?, cut, "{vector} {operation} rewrote a truncated store"); } } Ok(()) } + #[test] + fn refusals_say_what_was_found_without_blaming_variants() { + let unsupported = format!("{:#}", pull_vector("unknown-header.msgpack").unwrap_err()); + assert!(unsupported.contains("\"cultcache.store.v9\""), "{unsupported}"); + assert!(unsupported.contains("does not read"), "{unsupported}"); + assert!(!unsupported.contains("variant"), "{unsupported}"); + + // The C# reference refuses the same bytes: a v1 store with a longer record is + // not a store any runtime reads, so no upgrade is suggested. + let invalid = format!("{:#}", pull_vector("extra-slot-full-payload.msgpack").unwrap_err()); + assert!(invalid.contains("not a valid store"), "{invalid}"); + assert!(!invalid.contains("variant"), "{invalid}"); + assert!(!invalid.contains("runtime that"), "{invalid}"); + } + + /// The store path is a directory, so the staging file is written and synced and + /// then cannot be renamed over it: a failure after the write, reached without + /// permissions. + #[test] + fn a_write_that_fails_at_the_rename_leaves_no_staging_file() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + fs::create_dir(&path)?; + fs::write(path.join("kept"), b"kept")?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + + let error = store + .push_all(&[snapshot_envelope("alpha", b"1")], PushAllOptions::default()) + .unwrap_err(); + assert!(format!("{error:#}").contains("failed to atomically replace"), "{error:#}"); + + let staging: Vec<_> = fs::read_dir(temp.path())? + .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) + .collect::>>()? + .into_iter() + .filter(|name| name.starts_with("store.cc.") && name.ends_with(".tmp")) + .collect(); + assert_eq!(staging, Vec::::new()); + assert_eq!(fs::read(path.join("kept"))?, b"kept"); + Ok(()) + } + + /// MessagePack bytes of a fixstr, for store fixtures built by hand. + fn fixstr(text: &str) -> Vec { + assert!(text.len() < 32); + let mut bytes = vec![0xa0 | text.len() as u8]; + bytes.extend_from_slice(text.as_bytes()); + bytes + } + + #[test] + fn a_refusal_never_echoes_a_value_from_the_store() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + + // A v1 store whose catalog holds a string where an entry goes, and a legacy + // envelope array holding a string where an envelope goes: the decoder's own + // message would quote both strings. + let mut v1 = vec![0x93]; + v1.extend(fixstr(STORE_FORMAT_V1)); + v1.push(0x91); + v1.extend(fixstr("SECRET-CATALOG-TEXT")); + v1.push(0x90); + let mut legacy = vec![0x91]; + legacy.extend(fixstr("SECRET-LEGACY-TEXT")); + for bytes in [v1, legacy] { + fs::write(&path, &bytes)?; + for result in every_read(&store) { + let error = result.unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::Undecodable), "{error:#}"); + assert!(!format!("{error:#}").contains("SECRET"), "{error:#}"); + assert!(!format!("{error:?}").contains("SECRET"), "{error:?}"); + } + } + Ok(()) + } + #[test] fn a_rewrite_of_a_v1_store_stays_v1() -> Result<()> { assert_eq!( @@ -5188,4 +5733,306 @@ mod tests { ); Ok(()) } -} + + #[test] + fn a_header_is_echoed_only_in_the_known_shape() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + for (header, echoed) in [ + ("cultcache.store.v12", true), + ("cultcache.store.SECRET-HEADER", false), + ("cultcache.store.v12SECRET", false), + ("cultcache.store.v", false), + ] { + let mut bytes = vec![0x93]; + bytes.extend(fixstr(header)); + bytes.extend([0x90, 0x90]); + fs::write(&path, &bytes)?; + let error = store.pull_all().unwrap_err(); + let text = format!("{error:#}"); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{text}"); + if echoed { + assert!(text.contains(&format!("\"{header}\"")), "{text}"); + } else { + assert!(!text.contains(&format!("\"{header}\"")), "{text}"); + assert!(!text.contains("SECRET"), "{text}"); + assert!(text.contains(&format!("of {} bytes", header.len())), "{text}"); + } + } + Ok(()) + } + + /// Nothing at the store path is an empty store; a symbolic link whose target is + /// gone is something, and a writer must not rename a file over it. + #[cfg(unix)] + #[test] + fn a_dangling_symlink_at_the_store_path_is_an_error_and_nothing_is_written() -> Result<()> { + let temp = tempfile::tempdir()?; + let volume = temp.path().join("volume"); + fs::create_dir(&volume)?; + let target = volume.join("store.cc"); + let path = temp.path().join("store.cc"); + std::os::unix::fs::symlink(&target, &path)?; + let store = SingleFileMessagePackBackingStore::new(&path); + for result in every_read(&store) { + let error = result.expect_err("a dangling link must not read as an empty store"); + assert!(io_error_in(&error).is_some(), "{error:#}"); + assert_eq!(unreadable_kind(&error), None, "{error:#}"); + } + assert!(fs::symlink_metadata(&path)?.file_type().is_symlink()); + assert!(!target.exists()); + assert_eq!(fs::read_dir(&volume)?.count(), 0); + Ok(()) + } + + fn write_failure_kind(error: &anyhow::Error) -> Option { + error + .downcast_ref::() + .map(|failed| failed.kind) + } + + /// Runs `write` with `step` made to fail on this thread. + fn with_write_fault(step: WriteStep, write: impl FnOnce() -> T) -> T { + INJECTED_WRITE_FAULT.with(|fault| fault.set(Some(step))); + let result = write(); + INJECTED_WRITE_FAULT.with(|fault| fault.set(None)); + result + } + + #[test] + fn a_write_that_fails_before_the_rename_is_not_replaced_and_the_store_is_intact() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (store, before) = written_store(&path)?; + let error = with_write_fault(WriteStep::StagingSync, || { + store.compare_exchange_snapshot(&store.pull_all().unwrap(), &[snapshot_envelope("delta", b"4")]) + }) + .unwrap_err(); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert!(io_error_in(&error).is_some(), "{error:#}"); + assert!(error.to_string().contains(&format!("{} was not replaced", path.display())), "{error}"); + assert_eq!(fs::read(&path)?, before); + assert_eq!(fs::read_dir(temp.path())?.count(), 2, "only the store and its lock remain"); + + // The rename fails when the store path is a directory. + let blocked = temp.path().join("blocked.cc"); + fs::create_dir(&blocked)?; + let mut blocked_store = SingleFileMessagePackBackingStore::new(&blocked); + let error = blocked_store + .push_all(&[snapshot_envelope("alpha", b"1")], PushAllOptions::default()) + .unwrap_err(); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert!(fs::metadata(&blocked)?.is_dir()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_write_whose_directory_sync_fails_is_replaced_but_not_durable() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (store, _) = written_store(&path)?; + let replacement = vec![snapshot_envelope("delta", b"4")]; + let error = with_write_fault(WriteStep::DirectorySync, || { + store.compare_exchange_snapshot(&store.pull_all().unwrap(), &replacement) + }) + .unwrap_err(); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::ReplacedNotDurable), "{error:#}"); + assert!(error.to_string().contains(&format!("{} was replaced, but the replacement is not yet durable", path.display())), "{error}"); + assert!(io_error_in(&error).is_some(), "{error:#}"); + assert_eq!(store.pull_all()?, replacement); + Ok(()) + } + + #[test] + fn a_write_failure_names_no_value_from_the_snapshot() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let error = with_write_fault(WriteStep::StagingSync, || { + store.compare_exchange_snapshot(&[], &[snapshot_envelope("SECRET-KEY", b"SECRET-PAYLOAD")]) + }) + .unwrap_err(); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert!(!format!("{error:#}").contains("SECRET"), "{error:#}"); + Ok(()) + } + + #[test] + fn a_write_that_landed_is_reported_as_landed_when_the_unlock_after_it_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (mut store, _) = written_store(&path)?; + let replacement = vec![snapshot_envelope("delta", b"4")]; + let exchanged = with_write_fault(WriteStep::Unlock, || { + store.compare_exchange_snapshot(&store.pull_all().unwrap(), &replacement) + })?; + assert!(exchanged); + assert_eq!(store.pull_all()?, replacement); + + let pushed = snapshot_envelope("epsilon", b"5"); + with_write_fault(WriteStep::Unlock, || store.push(&pushed))?; + assert!(with_write_fault(WriteStep::Unlock, || store.pull_all())?.contains(&pushed)); + + // The lock was released: another writer takes it at once. + assert!(matches!( + store.try_compare_exchange_snapshot(&store.pull_all()?, &replacement)?, + TryCompareExchangeSnapshotOutcome::Exchanged + )); + + // The keyed store releases its lock the same way. + let mut keyed = RedbMessagePackBackingStore::new(temp.path().join("keyed.redb"))?; + with_write_fault(WriteStep::Unlock, || keyed.push(&pushed))?; + assert_eq!(keyed.pull_all()?, vec![pushed]); + Ok(()) + } + + // What a write does through a symbolic link today. R3 decides whether a write + // resolves the link or refuses it; until then these pin the current behaviour so the + // change is deliberate. Reads refuse a dangling link (see the test above). + #[cfg(unix)] + #[test] + fn r3_decides_resolve_or_refuse_a_push_through_a_live_link_replaces_the_link() -> Result<()> { + let temp = tempfile::tempdir()?; + let volume = temp.path().join("volume"); + fs::create_dir(&volume)?; + let target = volume.join("store.cc"); + SingleFileMessagePackBackingStore::new(&target).push(&snapshot_envelope("a", b"1"))?; + let before = fs::read(&target)?; + let link = temp.path().join("store.cc"); + std::os::unix::fs::symlink(&target, &link)?; + + SingleFileMessagePackBackingStore::new(&link).push(&snapshot_envelope("b", b"2"))?; + assert!(fs::symlink_metadata(&link)?.file_type().is_file(), "the link was replaced by a regular file"); + assert_eq!(fs::read(&target)?, before, "the store the link pointed to is unchanged"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn r3_decides_resolve_or_refuse_a_push_all_through_a_dangling_link_replaces_the_link() -> Result<()> { + let temp = tempfile::tempdir()?; + let volume = temp.path().join("volume"); + fs::create_dir(&volume)?; + let link = temp.path().join("store.cc"); + std::os::unix::fs::symlink(volume.join("store.cc"), &link)?; + + SingleFileMessagePackBackingStore::new(&link).push_all(&[snapshot_envelope("a", b"1")], PushAllOptions::default())?; + assert!(fs::symlink_metadata(&link)?.file_type().is_file(), "the link was replaced by a regular file"); + assert_eq!(fs::read_dir(&volume)?.count(), 0, "nothing reached the link's volume"); + Ok(()) + } + + #[test] + fn a_keyed_store_refusal_never_echoes_a_stored_value() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("keyed.redb"); + let envelope = snapshot_envelope("k", b"1"); + RedbMessagePackBackingStore::new(&path)?.push(&envelope)?; + { + let database = redb::Database::create(&path)?; + let write = database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + let keys: Vec> = table + .iter()? + .map(|row| row.map(|(key, _)| key.value().to_vec())) + .collect::>()?; + let mut value = vec![0xab]; + value.extend_from_slice(b"SECRET-REDB"); + for key in keys { + table.insert(key.as_slice(), value.as_slice())?; + } + } + write.commit()?; + } + let store = RedbMessagePackBackingStore::new(&path)?; + for error in [store.pull_all().unwrap_err(), store.compare_and_swap_entry(&envelope, snapshot_envelope("k", b"2")).unwrap_err()] { + assert!(format!("{error:#}").contains("failed to decode redb CultCache envelope"), "{error:#}"); + assert!(!format!("{error:#}").contains("SECRET"), "{error:#}"); + } + Ok(()) + } + + #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + enum Choice { + Only, + } + + #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + struct SkippedChoice { + #[serde(default, skip_serializing_if = "Option::is_none")] + choice: Option, + text: String, + } + + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "skipped-choice")] + struct ChoiceNote { + #[cultcache(key = 0)] + value: SkippedChoice, + } + + #[test] + fn a_document_decode_refusal_names_the_entry_and_never_a_stored_value() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + // A compact SkippedChoice with no choice is [text], so decoding it reads the text + // where the enum goes, and serde's own message would quote it as an unknown variant. + let secret = ChoiceNote { + value: SkippedChoice { + choice: None, + text: "SECRET-CHOICE".into(), + }, + }; + let mut payload = vec![0x91, 0x91, 0xad]; + payload.extend_from_slice(b"SECRET-CHOICE"); + let malformed = CultCacheEnvelope { + key: "k".into(), + r#type: ::TYPE.into(), + payload, + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: None, + }; + SingleFileMessagePackBackingStore::new(&path).push(&malformed)?; + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?; + cache.pull_all_backing_stores()?; + + let refusals = [ + ("get", cache.get::("k").unwrap_err()), + ("get_all", cache.get_all::().unwrap_err()), + ("get_all_with_keys", cache.get_all_with_keys::().unwrap_err()), + ("put_envelope", cache.put_envelope::(malformed.clone()).unwrap_err()), + ("prepare_entry", cache.prepare_entry("compact", &secret).unwrap_err()), + ]; + for (path, error) in refusals { + let text = format!("{error:#}"); + assert!(!text.contains("SECRET"), "{path}: {text}"); + assert!(text.contains("CultCache"), "{path}: {text}"); + } + Ok(()) + } + + #[test] + fn a_snapshot_the_store_cannot_hold_is_rejected_before_any_write() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (store, before) = written_store(&path)?; + let mut first = snapshot_envelope("x", b"1"); + first.schema_id = Some("shared".into()); + let mut second = snapshot_envelope("y", b"2"); + second.r#type = "other".into(); + second.schema_id = Some("shared".into()); + let error = store + .compare_exchange_snapshot(&store.pull_all()?, &[first, second]) + .unwrap_err(); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::Rejected), "{error:#}"); + assert!(error.to_string().contains("refused a snapshot it cannot store"), "{error}"); + assert!(io_error_in(&error).is_none(), "{error:#}"); + assert_eq!(fs::read(&path)?, before); + Ok(()) + } +} \ No newline at end of file diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 6da6ae1d..0cd77b41 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { lstat, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; @@ -44,7 +44,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } async pullAll(): Promise { - // The disk decides the header: a file that is gone, empty or legacy is not marked. + // The disk decides the header: a file that is gone or legacy is not marked. this.#format = STORE_FORMAT_VERSION; const disk = await this.#readDisk(); this.#format = disk.format; @@ -77,16 +77,8 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise { await this.#enqueue(async () => { - if (options.soft) { - try { - await readFile(this.filePath); - return; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "ENOENT") { - throw error; - } - } + if (options.soft && (await readStore(this.filePath)) !== undefined) { + return; } // A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids @@ -101,18 +93,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { // The one reader of the store file, asked by open, push, delete and flush alike. A file that is gone or zero bytes is an // empty unmarked store; anything else must decode completely or the read throws StoreUnreadableError. async #readDisk(): Promise { - let data: Uint8Array; - try { - data = await readFile(this.filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return { format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false }; - } - - throw error; - } - - if (data.length === 0) { + // An empty file is not a store, since no CultCache writer leaves one, so it is decoded and refused. + // Only a read that finds nothing at the path is an empty store. + const data = await readStore(this.filePath); + if (data === undefined) { return { format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false }; } @@ -156,6 +140,33 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } } +/** + * Reads a store file, or returns undefined when nothing is at its path. A dangling symbolic + * link is something: reading it as empty would let the writer rename a file over the link + * and move the store off its volume. Any other failure to reach the file is thrown. + */ +async function readStore(path: string): Promise { + try { + return await readFile(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + + try { + await lstat(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + + throw error; + } + + throw Object.assign(new Error(`CultCache store ${path} is a symbolic link whose target does not exist.`), { code: "ENOENT" }); +} + async function renameWithRetry(source: string, destination: string): Promise { let delayMs = 10; let lastError: unknown; diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index abf92487..c9573e81 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -24,8 +24,7 @@ export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...u if (decoded[0] !== STORE_FORMAT_VERSION && decoded[0] !== STORE_FORMAT_ELEMENT_IDS) { throw new Error( - `CultCache store format "${decoded[0]}" is not readable; this runtime reads "${STORE_FORMAT_VERSION}" and "${STORE_FORMAT_ELEMENT_IDS}" only. ` + - "The store needs a runtime that resolves document variants.", + `CultCache store format ${describeHeader(decoded[0])} is not one this runtime reads; it reads "${STORE_FORMAT_VERSION}" and "${STORE_FORMAT_ELEMENT_IDS}" only.`, ); } @@ -67,12 +66,41 @@ export class StoreUnreadableError extends Error { } } +/** + * A store header as a refusal may show it: echoed in the shape `cultcache.store.v`, + * and otherwise described only by its length, since the bytes are the store's. + */ +function describeHeader(header: string): string { + return /^cultcache\.store\.v[0-9]+$/u.test(header) + ? `"${header}"` + : `an unrecognised ${STORE_FORMAT_PREFIX}* header of ${new TextEncoder().encode(header).length} bytes`; +} + +/** + * A record key or schema id as a refusal may show it: a string is an identity and is named; + * anything else is described by its type, since the value is the store's. + */ +function describeIdentity(value: unknown): string { + if (typeof value === "string") { + return `"${value}"`; + } + + const kind = Array.isArray(value) + ? "array" + : value instanceof Uint8Array + ? "bytes" + : value === null + ? "nil" + : typeof value; + return `<${kind}>`; +} + /** Refuses a persisted record with more slots than v1 defines, naming its key and schema id. */ export function requireV1RecordSlots(record: unknown[]): void { if (record.length > PERSISTED_RECORD_SLOTS) { throw new Error( - `CultCache record "${String(record[0])}" (schema "${String(record[1])}") has ${record.length} slots; ` + - `this runtime reads ${PERSISTED_RECORD_SLOTS}. The store needs a runtime that resolves document variants.`, + `CultCache record ${describeIdentity(record[0])} (schema ${describeIdentity(record[1])}) has ${record.length} slots, more than ` + + `the ${PERSISTED_RECORD_SLOTS} of a ${STORE_FORMAT_VERSION} record, so this is not a valid store.`, ); } } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 2b33d4ae..008d8838 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { exec, execFile } from "node:child_process"; import { EventEmitter } from "node:events"; import { existsSync } from "node:fs"; -import { access, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { access, lstat, mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { dirname, join, resolve, delimiter } from "node:path"; import { test } from "node:test"; @@ -1988,6 +1988,129 @@ test("SingleFileMessagePackBackingStore refuses the C#-written variant store by ); }); +test("SingleFileMessagePackBackingStore refusals say what was found without blaming variants", async () => { + await assert.rejects( + () => pullVector("unknown-header.msgpack"), + (error: Error) => error.message.includes("not one this runtime reads") && !error.message.includes("variant"), + ); + await assert.rejects( + () => pullVector("extra-slot-full-payload.msgpack"), + (error: Error) => error.message.includes("not a valid store") && !error.message.includes("variant"), + ); +}); + +test("SingleFileMessagePackBackingStore reads only a missing store as empty", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-absent-")); + assert.deepEqual(await new SingleFileMessagePackBackingStore(join(dir, "store.cc")).pullAll(), []); + + // An existing empty file is not a store: no CultCache writer leaves one. + const empty = join(dir, "empty.cc"); + await writeFile(empty, new Uint8Array()); + await assert.rejects(() => new SingleFileMessagePackBackingStore(empty).pullAll()); + assert.equal((await readFile(empty)).length, 0); + + // An empty array is the legacy envelope array with no envelopes: no header, no format claimed. + const legacyEmpty = join(dir, "legacy-empty.cc"); + await writeFile(legacyEmpty, Uint8Array.of(0x90)); + assert.deepEqual(await new SingleFileMessagePackBackingStore(legacyEmpty).pullAll(), []); +}); + +test("SingleFileMessagePackBackingStore reports a store it cannot reach as an I/O error", { skip: process.platform === "win32" }, async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-unreachable-")); + const loop = join(dir, "loop.cc"); + await symlink(loop, loop); + const parent = join(dir, "body"); + await writeFile(parent, "not a directory"); + for (const [path, code] of [[loop, "ELOOP"], [join(parent, "store.cc"), "ENOTDIR"]] as const) { + await assert.rejects( + () => new SingleFileMessagePackBackingStore(path).pullAll(), + (error: NodeJS.ErrnoException) => error.code === code, + ); + } +}); + +test("SingleFileMessagePackBackingStore refusals never echo a value from the store", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-canary-")); + const file = join(dir, "store.cc"); + const cases: Array<[unknown[], boolean | undefined]> = [ + [["cultcache.store.v12", [], []], true], + [["cultcache.store.SECRET-HEADER", [], []], false], + [["cultcache.store.v12SECRET", [], []], false], + [["cultcache.store.v", [], []], false], + [["cultcache.store.v1", ["SECRET-CATALOG-TEXT"], []], undefined], + ]; + for (const [value, echoed] of cases) { + await writeFile(file, encode(value)); + const header = value[0] as string; + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pullAll(), + (error: Error) => { + if (echoed) { + return error.message.includes(`"${header}"`); + } + return !error.message.includes("SECRET") + && (echoed === undefined || (!error.message.includes(`"${header}"`) && error.message.includes(`of ${header.length} bytes`))); + }, + ); + } +}); + +test("SingleFileMessagePackBackingStore refuses a dangling symlink at its path and writes nothing", { skip: process.platform === "win32" }, async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-dangling-")); + const volume = join(dir, "volume"); + await mkdir(volume); + const file = join(dir, "store.cc"); + await symlink(join(volume, "store.cc"), file); + const store = new SingleFileMessagePackBackingStore(file); + const envelope = { key: "k", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + await assert.rejects(() => store.pullAll(), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); + await assert.rejects(() => store.push(envelope), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); + await assert.rejects(() => store.pushAll([envelope], { soft: true }), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); + assert.ok((await lstat(file)).isSymbolicLink()); + assert.deepEqual(await readdir(volume), []); +}); + +test("SingleFileMessagePackBackingStore record refusal names only a string key", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-record-key-")); + const file = join(dir, "store.cc"); + const secret = new TextEncoder().encode("SECRET-BYTES"); + for (const [key, schema] of [[["SECRET-IN-ARRAY"], "s"], [secret, "s"], ["k", { SECRET: 1 }]] as const) { + await writeFile(file, encode(["cultcache.store.v1", [], [[key, schema, "t", Uint8Array.of(0x90), "extra"]]])); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pullAll(), + (error: Error) => !error.message.includes("SECRET") && !error.message.includes("83,69,67") && error.message.includes("not a valid store"), + ); + } +}); + +// What a write does through a symbolic link today. R3 decides whether a write resolves the link +// or refuses it; these pin the current behaviour so that change is deliberate. +test("R3 decides resolve-or-refuse: a push through a live link replaces the link", { skip: process.platform === "win32" }, async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-live-link-")); + const volume = join(dir, "volume"); + await mkdir(volume); + const target = join(volume, "store.cc"); + const envelope = { key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + await new SingleFileMessagePackBackingStore(target).push(envelope); + const before = await readFile(target); + const link = join(dir, "store.cc"); + await symlink(target, link); + await new SingleFileMessagePackBackingStore(link).push({ ...envelope, key: "b" }); + assert.ok((await lstat(link)).isFile()); + assert.deepEqual(await readFile(target), before); +}); + +test("R3 decides resolve-or-refuse: a pushAll through a dangling link replaces the link", { skip: process.platform === "win32" }, async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-dangling-push-all-")); + const volume = join(dir, "volume"); + await mkdir(volume); + const link = join(dir, "store.cc"); + await symlink(join(volume, "store.cc"), link); + await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }]); + assert.ok((await lstat(link)).isFile()); + assert.deepEqual(await readdir(volume), []); +}); + test("CultCache inspector refuses the same vectors by name", async () => { const inspect = async (name: string) => inspectCultCacheBytes(name, await readFile(join(variantVectors, name))); await assert.rejects(() => inspect("unknown-header.msgpack"), /cultcache\.store\.v9/u); diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 93283615..10cc2a58 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -11,6 +11,7 @@ import java.io.InputStream import java.io.OutputStream import java.net.DatagramPacket import java.net.DatagramSocket +import java.net.Inet6Address import java.net.InetAddress import java.net.InetSocketAddress import java.net.ServerSocket @@ -604,7 +605,7 @@ object CultMesh { connectionId: Long, remoteHost: String, remotePort: Int, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = cultNetRudpClient( @@ -623,7 +624,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: CultNetRudpEndpoint, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = createRudpClient( @@ -640,7 +641,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: String, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection = createRudpClient( @@ -656,7 +657,7 @@ object CultMesh { runtimeId: String, connectionId: Long, peer: CultMeshPeerCard, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { @@ -674,7 +675,7 @@ object CultMesh { role: String, shardId: String? = null, at: Instant = Instant.now(), - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { @@ -687,7 +688,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: CultNetRudpEndpoint, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -706,7 +707,7 @@ object CultMesh { runtimeId: String, connectionId: Long, endpoint: String, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -728,7 +729,7 @@ object CultMesh { runtimeId: String, connectionId: Long, peer: CultMeshPeerCard, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -759,7 +760,7 @@ object CultMesh { role: String, shardId: String? = null, at: Instant = Instant.now(), - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), connectPayload: ByteArray = ByteArray(0), @@ -3051,22 +3052,36 @@ fun cultNetRudpServer( ) } +/** + * The address an RUDP client binds when its caller names none: loopback for a + * loopback endpoint, otherwise the unspecified address of the endpoint's family. + * A socket bound to loopback cannot send off the host on Windows. + */ +fun cultNetRudpClientBindAddress(remote: InetAddress): InetAddress = when { + remote is Inet6Address && remote.isLoopbackAddress -> InetAddress.getByName("::1") + remote is Inet6Address -> InetAddress.getByName("::") + remote.isLoopbackAddress -> InetAddress.getByName("127.0.0.1") + else -> InetAddress.getByName("0.0.0.0") +} + fun cultNetRudpClient( runtimeId: String, connectionId: Long, remoteHost: String, remotePort: Int, - bindHost: String = "127.0.0.1", + bindHost: String? = null, bindPort: Int = 0, tuning: CultNetRudpSocketTuning = CultNetRudpSocketTuning(), ): CultNetRudpSocketTransportConnection { - val socket = DatagramSocket(bindPort, InetAddress.getByName(bindHost)).also { it.soTimeout = 20 } + val remote = InetAddress.getByName(remoteHost) + val bindAddress = bindHost?.let(InetAddress::getByName) ?: cultNetRudpClientBindAddress(remote) + val socket = DatagramSocket(bindPort, bindAddress).also { it.soTimeout = 20 } return CultNetRudpSocketTransportConnection( socket = socket, mode = CultNetRudpSocketMode.Client, runtimeId = runtimeId, connectionId = connectionId, - remoteAddress = InetSocketAddress(InetAddress.getByName(remoteHost), remotePort), + remoteAddress = InetSocketAddress(remote, remotePort), initialSequence = tuning.initialSequence, resendDelayMs = tuning.resendDelayMs, maxFragmentBytes = tuning.maxFragmentBytes, @@ -3391,6 +3406,7 @@ fun main(args: Array) { rudpSessionFragmentsAndReassemblesReliableOrderedPayloads() rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() + rudpClientBindsLoopbackOnlyForALoopbackEndpoint() rudpSocketTransportHandshakesAndCarriesReliableOrderedSchemaFrames() rudpSocketTransportCarriesFragmentedReliableOrderedSchemaFrames() return @@ -5437,6 +5453,27 @@ private fun rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() { check(oldAck.ackMask == 0L) } +private fun rudpClientBoundHost(endpoint: String, bindHost: String? = null): InetAddress = + CultMesh.createRudpClient("kotlin-rudp-bind-client", 0x10203060L, endpoint, bindHost).use { client -> + InetAddress.getByName(client.profile.transports.single().host) + } + +// A client bound to loopback cannot send to another host on Windows, so a +// client for a remote endpoint binds the unspecified address. A JVM wildcard +// socket is dual-stack and reports one wildcard for either family, so the +// family is not observable here. +private fun rudpClientBindsLoopbackOnlyForALoopbackEndpoint() { + check(rudpClientBoundHost("rudp://10.77.0.1:17872").isAnyLocalAddress) + check(rudpClientBoundHost("rudp://[2001:db8::1]:17872").isAnyLocalAddress) + check(rudpClientBoundHost("rudp://127.0.0.1:17872") == InetAddress.getByName("127.0.0.1")) + check(rudpClientBoundHost("rudp://[::1]:17872") == InetAddress.getByName("::1")) + check(rudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1") == InetAddress.getByName("127.0.0.1")) + check(rudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0").isAnyLocalAddress) + CultMesh.createRudpServer("kotlin-rudp-bind-server", 0x10203061L).use { server -> + check(InetAddress.getByName(server.profile.transports.single().host) == InetAddress.getByName("127.0.0.1")) + } +} + private fun rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() { val connectionId = 0x10203042L CultMesh.createRudpServer( diff --git a/packages/cultmesh-py/src/cultmesh_py/facade.py b/packages/cultmesh-py/src/cultmesh_py/facade.py index 53328aab..fea60610 100644 --- a/packages/cultmesh-py/src/cultmesh_py/facade.py +++ b/packages/cultmesh-py/src/cultmesh_py/facade.py @@ -18,6 +18,7 @@ from cultnet_py import ( CultNetShardCatalog, CultNetSimulationObservationHub, create_rudp_schema_transport, + rudp_client_bind_host, wire_message_schema_catalog, ) @@ -157,7 +158,7 @@ class CultMesh: bind_host: str = "127.0.0.1", bind_port: int = 0, socket: socket_module.socket | None = None, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 250, transport_id: str = "rudp", max_payload_bytes: int | None = None, @@ -188,10 +189,10 @@ class CultMesh: connection_id: int, endpoint: str | CultMeshRudpEndpoint, *, - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, socket: socket_module.socket | None = None, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 250, transport_id: str = "rudp", max_payload_bytes: int | None = None, @@ -204,7 +205,10 @@ class CultMesh: if isinstance(endpoint, str) else endpoint ) - transport_socket = socket or _bind_rudp_socket(bind_host, bind_port) + transport_socket = socket or _bind_rudp_socket( + bind_host if bind_host is not None else rudp_client_bind_host(parsed_endpoint.host), + bind_port, + ) return CultNetRudpSocketTransportConnection( CultNetRudpSocketTransportOptions( runtime_id=runtime_id, @@ -429,7 +433,7 @@ class CultMesh: endpoint: str | CultMeshRudpEndpoint | None = None, connection_id: int = 0x43554C54, runtime_id: str = "cultmesh-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, ) -> CultNetRawClient: if endpoint is None and host.lower().startswith("rudp://"): @@ -465,7 +469,7 @@ class CultMesh: endpoint: str | CultMeshRudpEndpoint | None = None, connection_id: int = 0x43554C54, runtime_id: str = "cultmesh-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, ) -> CultNetRawClient: return CultMesh.create_client( @@ -481,7 +485,10 @@ class CultMesh: def _bind_rudp_socket(bind_host: str, bind_port: int) -> socket_module.socket: - transport_socket = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_DGRAM) + transport_socket = socket_module.socket( + socket_module.AF_INET6 if ":" in bind_host else socket_module.AF_INET, + socket_module.SOCK_DGRAM, + ) transport_socket.bind((bind_host, bind_port)) transport_socket.settimeout(0.02) return transport_socket diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index e0c7fdd0..fe4edd52 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -224,15 +224,19 @@ class CultMeshLocalServer: now_ms = _now_ms() peer = peers.get(remote_addr) if packet.packet_type == CultNetRudpPacketType.CONNECT: - peer = _RudpPeerConnection( - CultNetRudpSession( - CultNetRudpSessionOptions( - connection_id=self.rudp_connection_id, - resend_delay_ms=self.rudp_resend_delay_ms, + # The session decides whether this Connect repeats the one it + # accepted. Any other Connect is a new client session, with new + # subscriptions. + if peer is None or not peer.session.connect_repeats(packet): + peer = _RudpPeerConnection( + CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=self.rudp_connection_id, + resend_delay_ms=self.rudp_resend_delay_ms, + ) ) ) - ) - peers[remote_addr] = peer + peers[remote_addr] = peer self._send_rudp_packet( rudp_socket, remote_addr, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 5e2befcf..82edd616 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -213,6 +213,37 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(received_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") self.assertEqual(received_messages[0]["messageId"], "rudp-transport-factory") + def _rudp_client_bound_host(self, endpoint: str, bind_host: str | None = None) -> str: + client = CultMesh.create_rudp_client("python-cultmesh-bind-client", 0x10203060, endpoint, bind_host=bind_host) + try: + return client.socket.getsockname()[0] + finally: + client.socket.close() + + # A client bound to loopback cannot send to another host on Windows, so a + # client for a remote endpoint binds the unspecified address of its family. + def test_cultmesh_rudp_client_for_a_remote_endpoint_binds_the_unspecified_address(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872"), "0.0.0.0") + self.assertEqual(self._rudp_client_bound_host("rudp://[2001:db8::1]:17872"), "::") + + def test_cultmesh_rudp_client_for_a_loopback_endpoint_binds_loopback(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872"), "127.0.0.1") + self.assertEqual(self._rudp_client_bound_host("rudp://[::1]:17872"), "::1") + + def test_cultmesh_rudp_client_binds_an_explicit_bind_host_whatever_the_endpoint(self) -> None: + self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872", "127.0.0.1"), "127.0.0.1") + self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872", "0.0.0.0"), "0.0.0.0") + + def test_cultmesh_rudp_server_binds_loopback_unless_told_otherwise(self) -> None: + loopback = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203061) + anywhere = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203062, bind_host="0.0.0.0") + try: + self.assertEqual(loopback.socket.getsockname()[0], "127.0.0.1") + self.assertEqual(anywhere.socket.getsockname()[0], "0.0.0.0") + finally: + loopback.socket.close() + anywhere.socket.close() + def test_cultmesh_facade_creates_rudp_client_from_peer_endpoint(self) -> None: connection_id = 0x10203042 server = CultMesh.create_rudp_server( diff --git a/packages/cultmesh-py/tests/test_rudp_server_generation.py b/packages/cultmesh-py/tests/test_rudp_server_generation.py new file mode 100644 index 00000000..967a03c2 --- /dev/null +++ b/packages/cultmesh-py/tests/test_rudp_server_generation.py @@ -0,0 +1,159 @@ +"""The CultMesh local server's RUDP loop asks each peer's session whether a Connect +repeats. A retransmitted Connect keeps the peer; a restarted client on the same +address is a new one, with none of the old one's subscriptions.""" +from __future__ import annotations + +import socket +import time +import unittest +from dataclasses import dataclass + +import msgpack +from cultcache_py import define_database_entry_type +from cultmesh_py import create_node +from cultmesh_py.server import CultMeshLocalServer +from cultnet_py import ( + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + document_put_raw, +) +from cultnet_py.transport import CultNetRudpPacketType, decode_rudp_packet, encode_rudp_packet + +CONNECTION_ID = 0x10203091 +SCHEMA_ID = "mesh.rudp_generation_note.v1" +RELIABLE = CultNetRudpSendOptions(reliable=True, ordered=True) + + +@dataclass +class Note: + body: str + + +NOTE = define_database_entry_type( + "mesh.rudp_generation_note", + [("body", 0)], + cls=Note, + schema_id=SCHEMA_ID, + schema_name="mesh.rudp_generation_note", + schema_version=SCHEMA_ID, +) + + +def wire(message: dict) -> bytes: + return msgpack.packb(message, use_bin_type=True) + + +class RudpServerGenerationTests(unittest.TestCase): + def setUp(self) -> None: + node = create_node(runtime_id="rudp-generation") + node.database.register_document(NOTE) + self.server = CultMeshLocalServer(node=node, rudp_connection_id=CONNECTION_ID) + self.server.start() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.bind(("127.0.0.1", 0)) + self.sock.settimeout(0.02) + self.target = ("127.0.0.1", self.server.port) + # Every schema data packet the server sent, by sequence, whether or not the receiving + # session delivered it: a new server-side session numbers its packets afresh. + self.sent: dict[int, str] = {} + + def tearDown(self) -> None: + self.sock.close() + self.server.stop() + + def to_server(self, packet) -> None: + self.sock.sendto(encode_rudp_packet(packet), self.target) + + def pump(self, session: CultNetRudpSession, until, seconds: float = 2.0) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if until(): + return True + try: + data, _ = self.sock.recvfrom(65535) + except TimeoutError: + continue + packet = decode_rudp_packet(data) + session.receive(packet, int(time.monotonic() * 1000)) + if packet.reliable or packet.packet_type == CultNetRudpPacketType.ACCEPT: + self.to_server(session.create_ack_for_received(packet.sequence)) + if packet.packet_type == CultNetRudpPacketType.DATA and packet.channel_id == "schema": + self.sent[packet.sequence] = msgpack.unpackb(packet.payload, raw=False).get("schemaVersion") + return until() + + def count(self, schema_version: str) -> int: + return sum(1 for value in self.sent.values() if value == schema_version) + + def new_session(self, initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence)) + + def test_a_retransmitted_connect_keeps_the_peer_and_a_restarted_client_is_a_new_one(self) -> None: + first = self.new_session(50_000) + connect = first.create_connect(0, b"same") + self.to_server(connect) + self.assertTrue(self.pump(first, lambda: first.connected), "the first Accept") + hello = first.send( + "schema", wire({"schemaVersion": "cultnet.hello.v0", "runtimeId": "probe", "runtimeKind": "test"}), RELIABLE + ) + self.to_server(hello) + self.assertTrue(self.pump(first, lambda: self.count("cultnet.hello.v0") == 1), "the first answer") + self.to_server( + first.send( + "schema", + wire({ + "schemaVersion": "cultnet.database_subscribe.v0", + "messageId": "subscribe", + "subscriptionId": "old-incarnation", + "schemaIds": [SCHEMA_ID], + "includeSnapshot": False, + }), + RELIABLE, + ) + ) + self.pump(first, lambda: False, 0.2) + first_put = document_put_raw( + message_id="first-put", + key="note:a", + schema_id=SCHEMA_ID, + stored_at="2026-09-30T00:00:00Z", + payload=NOTE.encode_payload(Note("first")), + ) + self.to_server(first.send("schema", first_put.to_bytes(), RELIABLE)) + self.assertTrue( + self.pump(first, lambda: self.count("cultnet.database_change_raw.v0") == 1), + "the subscription did not notify its own connection", + ) + + # A retransmitted Connect repeats: the data already delivered is not delivered, or answered, again. + self.to_server(connect) + self.to_server(hello) + self.pump(first, lambda: False, 0.4) + self.assertEqual(self.count("cultnet.hello.v0"), 1, "the retransmitted Connect reset the peer") + + # The same client restarted on the same address, connection id and payload: a new peer. + second = self.new_session(7) + self.to_server(second.create_connect(0, b"same")) + self.assertTrue(self.pump(second, lambda: second.connected), "the restarted client's Accept") + self.to_server( + second.send( + "schema", wire({"schemaVersion": "cultnet.hello.v0", "runtimeId": "probe", "runtimeKind": "test"}), RELIABLE + ) + ) + self.assertTrue(self.pump(second, lambda: self.count("cultnet.hello.v0") == 2), "the restarted client's answer") + + # It inherited none of the old incarnation's subscriptions. + put = document_put_raw( + message_id="second-put", + key="note:a", + schema_id=SCHEMA_ID, + stored_at="2026-09-30T00:00:01Z", + payload=NOTE.encode_payload(Note("second")), + ) + self.to_server(second.send("schema", put.to_bytes(), RELIABLE)) + self.pump(second, lambda: False, 0.4) + self.assertEqual(self.count("cultnet.database_change_raw.v0"), 1, "the new peer inherited a subscription") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 88a6aaad..2c5d33ce 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -961,7 +961,7 @@ fn connect_rudp_client( mode: cultnet_rs::CultNetRudpSocketMode::Client, remote_addr: Some(target), connection_id: fresh_rudp_connection_epoch(), - initial_sequence: 1, + initial_sequence: None, resend_delay_ms, transport_id: Some(transport_id.to_string()), max_payload_bytes: None, @@ -1406,7 +1406,7 @@ mod tests { let mut session = cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { connection_id: connect.connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 5, max_pending_reliable_packets: Some(64), }); diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index b99c7145..47348bfe 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -410,12 +410,20 @@ where packet: &CultNetRudpPacket, now: u64, ) -> Result { + // The session decides whether a Connect repeats the one it accepted. Any + // other Connect from this key is a new client session: the old one ends + // with its budget, and the new one starts fresh. + if self + .sessions + .get(&key) + .is_some_and(|entry| !entry.session.connect_repeats(packet)) + { + self.sessions.remove(&key); + } let reply = match self.sessions.get_mut(&key) { - // The same key is a retransmitted Connect for the existing epoch. - // A genuinely fresh client incarnation must choose a fresh id. Some(entry) => { entry.last_activity_monotonic_millis = now; - match entry.session.answer_repeated_connect(packet, now) { + match entry.session.accept_connect(packet, now, Vec::new()) { Ok(reply) => reply, Err(_) => { self.end_refused_session(key)?; @@ -429,7 +437,7 @@ where } let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: key.connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: duration_millis(self.options.resend_delay), max_pending_reliable_packets: Some( self.options.max_pending_reliable_packets_per_session, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index fa741827..e43e4e59 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -160,7 +160,15 @@ fn server( } fn client(target: SocketAddr, id: u32) -> Result { - let socket = UdpSocket::bind("127.0.0.1:0")?; + client_on(UdpSocket::bind("127.0.0.1:0")?, target, id, 1) +} + +fn client_on( + socket: UdpSocket, + target: SocketAddr, + id: u32, + initial_sequence: u32, +) -> Result { socket.set_nonblocking(true)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_delivery: None, @@ -169,7 +177,7 @@ fn client(target: SocketAddr, id: u32) -> Result Result<()> } #[test] -fn duplicate_connect_preserves_epoch_and_fresh_epoch_is_separate() -> Result<()> { +fn a_second_connect_from_a_client_replaces_its_session_and_a_fresh_epoch_is_separate() -> Result<()> { let clock = Clock::new(45_000); let sink = Sink::default(); let mut server = server(Default::default(), clock, sink.clone(), Source::default())?; @@ -325,6 +333,97 @@ fn duplicate_connect_preserves_epoch_and_fresh_epoch_is_separate() -> Result<()> Ok(()) } +/// A client that restarts on the same address and connection id starts a new +/// session: its Connect carries a sequence the accepted one did not, so the +/// server does not mistake it for a retransmit, and what it sends is delivered. +#[test] +fn a_restarted_client_on_the_same_address_and_connection_id_is_admitted() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(46_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 93, 50_000)?; + connect(&mut server, &mut [&mut first])?; + for index in 0..3 { + send( + &mut first, + &CultNetMessage::DocumentPutRaw { + message_id: format!("before-{index}"), + document: document("before", vec![index]), + }, + )?; + } + for _ in 0..40 { + server.poll_once()?; + first.receive_once()?; + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 3); + + let mut restarted = client_on(twin, target, 93, 7)?; + connect(&mut server, &mut [&mut restarted])?; + assert_eq!(server.session_count(), 1); + send( + &mut restarted, + &CultNetMessage::DocumentPutRaw { + message_id: "after-restart".into(), + document: document("after", vec![9]), + }, + )?; + for _ in 0..40 { + server.poll_once()?; + restarted.receive_once()?; + } + let receipts = sink.0.lock().unwrap(); + assert!( + receipts.receipts.iter().any(|r| r.message_id == "after-restart"), + "the restarted client's frame was not delivered" + ); + Ok(()) +} + +/// A restarted client is a new session, and starts with a new payload budget: +/// what the old one spent is not charged to it. +#[test] +fn a_restarted_client_starts_with_a_fresh_payload_budget() -> Result<()> { + let put = |id: &str| CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document("budget", vec![7; 64]), + }; + let encoded = encode_cultnet_message_to_vec(&put("put-a"), CultNetWireContract::CultNetSchemaV0)?; + let options = CultMeshRudpDocumentServerOptions { + max_admitted_payload_bytes: encoded.len() * 2, + max_admitted_payload_bytes_per_session: encoded.len(), + max_snapshot_response_bytes: encoded.len(), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(47_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 94, 50_000)?; + connect(&mut server, &mut [&mut first])?; + send(&mut first, &put("put-a"))?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + + let mut restarted = client_on(twin, target, 94, 7)?; + connect(&mut server, &mut [&mut restarted])?; + send(&mut restarted, &put("put-b"))?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!( + sink.0.lock().unwrap().receipts.len(), + 2, + "the restarted client was charged what the old session spent" + ); + Ok(()) +} + #[test] fn application_rejection_is_nonfatal_peer_scoped_and_unacknowledged() -> Result<()> { let clock = Clock::new(48_000); @@ -576,7 +675,7 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { let session = |id| { CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 10, max_pending_reliable_packets: None, }) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index ff7140d3..288c8ebc 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -16,6 +16,7 @@ import { CultNetRudpSession, CultNetPeer, CultNetRudpSocketTransportConnection, + rudpClientBindHost, CultNetSchemaCatalog, CultNetShardCatalog, cultNetBuiltinSchemaRegistry, @@ -4191,6 +4192,11 @@ export interface CultMeshRudpEndpoint { } export interface CultMeshRudpSocketOptions { + /** + * Local address to bind. Unset, a server binds loopback, and a client binds + * loopback for a loopback endpoint and the unspecified address of the + * endpoint's family otherwise, so it can reach a remote host. + */ bindHost?: string; bindPort?: number; socket?: Socket; @@ -5654,7 +5660,7 @@ export class CultMesh { if (packet.packetType === "connect") { const connectPayload = Uint8Array.from(packet.payload ?? []); - if (record && byteArraysEqual(record.connectPayload, connectPayload)) { + if (record?.session.connectRepeats(packet)) { socket.send( encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), remote.port, @@ -5707,7 +5713,7 @@ export class CultMesh { } if (packet.reliable) { socket.send( - encodeRudpPacket(record.session.createAckFor(packet.sequence)), + encodeRudpPacket(record.session.createAckForReceived(packet.sequence)), record.remote.port, record.remote.address, ); @@ -5996,7 +6002,7 @@ export class CultMesh { options: CultMeshRudpSocketOptions = {}, ): Promise { requireNonEmpty(runtimeId, "runtimeId"); - const socket = options.socket ?? (await bindRudpSocket(options)); + const socket = options.socket ?? (await bindRudpSocket(options.bindHost ?? "127.0.0.1", options)); return new CultNetRudpSocketTransportConnection({ runtimeId, socket, @@ -6022,7 +6028,10 @@ export class CultMesh { requireNonEmpty(runtimeId, "runtimeId"); const parsedEndpoint = typeof endpoint === "string" ? CultMesh.parseRudpEndpoint(endpoint) : endpoint; - const socket = options.socket ?? (await bindRudpSocket(options)); + const socket = options.socket ?? (await bindRudpSocket( + options.bindHost ?? (await rudpClientBindHost(parsedEndpoint.host)), + options, + )); return new CultNetRudpSocketTransportConnection({ runtimeId, socket, @@ -6419,9 +6428,8 @@ function toUint8Array(value: unknown): Uint8Array { throw new Error("CultNet raw document payload was not binary."); } -async function bindRudpSocket(options: CultMeshRudpSocketOptions): Promise { - const socket = createSocket("udp4"); - const host = options.bindHost ?? "127.0.0.1"; +async function bindRudpSocket(host: string, options: CultMeshRudpSocketOptions): Promise { + const socket = createSocket(host.includes(":") ? "udp6" : "udp4"); const port = options.bindPort ?? 0; await new Promise((resolve, reject) => { socket.once("error", reject); @@ -6504,14 +6512,6 @@ function nonBlankOr(value?: string, fallback = ""): string { return value && value.trim() ? value : fallback; } -function byteArraysEqual(left: Uint8Array, right: Uint8Array): boolean { - if (left.byteLength !== right.byteLength) return false; - for (let index = 0; index < left.byteLength; index += 1) { - if (left[index] !== right[index]) return false; - } - return true; -} - function parseCultMeshLocalityKind( value: string | undefined, fallback: CultMeshLocalityKind, diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 99d1adc2..7c091f06 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -2635,6 +2635,44 @@ test("CultMesh TS branded facade exposes schema and shard catalogs", () => { ); }); +async function rudpClientBoundHost(endpoint: string, bindHost?: string): Promise { + const client = await CultMesh.createRudpClient("cultmesh-ts-bind-client", 0x10203060, endpoint, { bindHost }); + try { + return client.profile.transports[0]?.host; + } finally { + client.close(); + } +} + +// A client bound to loopback cannot send to another host on Windows, so a +// client for a remote endpoint binds the unspecified address of its family. +test("CultMesh TS RUDP client for a remote endpoint binds the unspecified address", async () => { + assert.equal(await rudpClientBoundHost("rudp://10.77.0.1:17872"), "0.0.0.0"); + assert.equal(await rudpClientBoundHost("rudp://[2001:db8::1]:17872"), "::"); +}); + +test("CultMesh TS RUDP client for a loopback endpoint binds loopback", async () => { + assert.equal(await rudpClientBoundHost("rudp://127.0.0.1:17872"), "127.0.0.1"); + assert.equal(await rudpClientBoundHost("rudp://[::1]:17872"), "::1"); +}); + +test("CultMesh TS RUDP client binds an explicit bindHost whatever the endpoint", async () => { + assert.equal(await rudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1"), "127.0.0.1"); + assert.equal(await rudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0"), "0.0.0.0"); +}); + +test("CultMesh TS RUDP server binds loopback unless told otherwise", async () => { + const loopback = await CultMesh.createRudpServer("cultmesh-ts-bind-server", 0x10203061); + const any = await CultMesh.createRudpServer("cultmesh-ts-bind-server", 0x10203062, { bindHost: "0.0.0.0" }); + try { + assert.equal(loopback.profile.transports[0]?.host, "127.0.0.1"); + assert.equal(any.profile.transports[0]?.host, "0.0.0.0"); + } finally { + loopback.close(); + any.close(); + } +}); + test("CultMesh TS branded facade creates RUDP clients from peer endpoints", async () => { const connectionId = 0x10203044; const server = await CultMesh.createRudpServer( diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index 5b44c84b..f1dd5d2d 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -8,6 +8,7 @@ import { decodeRudpPacket, encodeRudpPacket, type CultNetOperationRequestMessage, + type CultNetRudpPacket, } from "cultnet-ts"; import { @@ -320,6 +321,80 @@ test("a replaced RUDP generation drops operations queued behind an active handle } }); +test("a client restarted on the same address with the same Connect payload replaces its RUDP session, and a retransmitted Connect does not", async () => { + const connectionId = 0x43554c54; + let closedSessions = 0; + const server = CultMesh.createRudpDocumentServer("odin-restart-test", connectionId, { + bindPort: 0, + documents: new CultNetDocumentRegistry(), + onSessionClosed: () => { closedSessions += 1; }, + onError: () => undefined, + }); + await server.start(); + const socket = createSocket("udp4"); + const received: CultNetRudpPacket[] = []; + socket.on("message", wire => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), server.bind.port, "127.0.0.1"); + const evidence = Buffer.from("same connect payload"); + try { + const first = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = first.createConnect(Date.now(), evidence); + toServer(connect); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the first Accept"); + + // A retransmitted Connect repeats: the session stays. + toServer(connect); + await new Promise(resolve => setTimeout(resolve, 100)); + assert.equal(closedSessions, 0, "a retransmitted Connect closed the session"); + + // The same client restarted: another sequence, the same payload. + received.length = 0; + const second = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(second.createConnect(Date.now(), evidence)); + await waitFor(() => closedSessions === 1, "the old RUDP session to close"); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the restarted client's Accept"); + } finally { + socket.close(); + server.close(); + } +}); + +test("the RUDP document server does not acknowledge by name a reliable packet its session did not receive", async () => { + const connectionId = 0x43554c55; + const server = CultMesh.createRudpDocumentServer("odin-ack-test", connectionId, { + bindPort: 0, + documents: new CultNetDocumentRegistry(), + onError: () => undefined, + }); + await server.start(); + const socket = createSocket("udp4"); + const received: CultNetRudpPacket[] = []; + socket.on("message", wire => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), server.bind.port, "127.0.0.1"); + try { + const client = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = client.createConnect(Date.now()); + toServer(connect); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the Accept"); + received.length = 0; + + // A reliable packet the session ignores (a client does not send an Accept). + // The session never received it, so the Ack must not name it. + const ignored: CultNetRudpPacket = { ...client.createAck(), packetType: "accept", reliable: true, sequence: connect.sequence + 5 }; + toServer(ignored); + await waitFor(() => received.some(packet => packet.packetType === "ack"), "the acknowledgement"); + const ack = received.find(packet => packet.packetType === "ack")!; + const names = ack.ack === ignored.sequence + || Array.from({ length: 32 }, (_, bit) => bit).some(bit => (ack.ackMask & (1 << bit)) !== 0 && ack.ack - bit - 1 === ignored.sequence); + assert.equal(names, false, "the server acknowledged by name a packet its session refused"); + } finally { + socket.close(); + server.close(); + } +}); + function deferred(): { promise: Promise; resolve(value: T): void } { let resolve!: (value: T) => void; return { promise: new Promise(done => resolve = done), resolve }; diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 61c81668..581a8f47 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -6,6 +6,7 @@ from .client import ( CultNetSchemaTransportFactory, create_rudp_schema_transport, create_tcp_framed_schema_transport, + rudp_client_bind_host, ) from .cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, @@ -228,6 +229,7 @@ __all__ = [ "compute_reconnect_delay_ms", "create_reconnect_policy", "create_rudp_schema_transport", + "rudp_client_bind_host", "create_tcp_framed_schema_transport", "create_rudp_transport_profile", "create_tcp_framed_transport_profile", diff --git a/packages/cultnet-py/src/cultnet_py/client.py b/packages/cultnet-py/src/cultnet_py/client.py index c30859d9..28060753 100644 --- a/packages/cultnet-py/src/cultnet_py/client.py +++ b/packages/cultnet-py/src/cultnet_py/client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ipaddress import socket import time from collections.abc import Callable @@ -384,6 +385,22 @@ def create_tcp_framed_schema_transport( ) +def rudp_client_bind_host(endpoint_host: str) -> str: + """Return the address an RUDP client binds when its caller names none. + + Loopback for a loopback endpoint, otherwise the unspecified address of the + endpoint's family. A socket bound to loopback cannot send off the host on + Windows. + """ + family, _, _, _, address = socket.getaddrinfo( + endpoint_host.removeprefix("[").removesuffix("]"), None, type=socket.SOCK_DGRAM + )[0] + loopback = ipaddress.ip_address(address[0]).is_loopback + if family == socket.AF_INET6: + return "::1" if loopback else "::" + return "127.0.0.1" if loopback else "0.0.0.0" + + def create_rudp_schema_transport( *, host: str, @@ -391,16 +408,20 @@ def create_rudp_schema_transport( connection_id: int, timeout_seconds: float = 4.0, runtime_id: str = "cultnet-python-rudp-client", - bind_host: str = "127.0.0.1", + bind_host: str | None = None, bind_port: int = 0, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 25, transport_id: str = "rudp", max_payload_bytes: int | None = None, max_fragment_bytes: int | None = None, max_pending_reliable_packets: int | None = None, ) -> CultNetRudpSocketTransportConnection: - transport_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if bind_host is None: + bind_host = rudp_client_bind_host(host) + transport_socket = socket.socket( + socket.AF_INET6 if ":" in bind_host else socket.AF_INET, socket.SOCK_DGRAM + ) transport_socket.bind((bind_host, bind_port)) transport_socket.settimeout(min(timeout_seconds, 0.02)) transport = CultNetRudpSocketTransportConnection( diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 0f378613..355faa51 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +import secrets import socket import threading import time @@ -292,10 +293,45 @@ class CultNetRudpReceiveResult: disconnect_reason: bytes = b"" +# How long a client keeps retransmitting a Connect nobody answered before it +# abandons that attempt for a fresh one. A Connect the server answers with an +# Ack, never an Accept, is one the server judged stale (see `accept_connect`); +# retransmitting the same sequence would never change its mind. +_RUDP_CONNECT_ATTEMPT_MS = 3_000 + + +def _draw_sequence() -> int: + """A sequence drawn from a secure source in [1, 2^31). The Connect's + sequence is what tells a peer whether a Connect repeats one it already + accepted or starts a new session, so two sessions must not share one by + default.""" + return 1 + secrets.randbelow(2**31 - 1) + + +def _at_or_before(sequence: int, mark: int, window: int) -> bool: + """Whether `sequence` is at or before `mark` in serial order, within the + receive window. The compare is modular, so it holds across the wrap of the + 32-bit space; a sequence further back than the window is the duplicate + test's below-window clause, and one ahead of the mark is never before it.""" + return ((mark - sequence) & 0xFFFFFFFF) < window + + +def _packet_acknowledges(packet: "CultNetRudpPacket", sequence: int) -> bool: + if packet.ack == sequence: + return True + return any( + packet.ack_mask & (1 << bit) and packet.ack > bit and packet.ack - bit - 1 == sequence + for bit in range(32) + ) + + @dataclass(frozen=True) class CultNetRudpSessionOptions: connection_id: int - initial_sequence: int = 1 + # The first sequence this session issues. Unset, the session draws its own + # at random: every session is a new identity to its peer, and an options + # object reused for several must not give them one sequence. + initial_sequence: int | None = None resend_delay_ms: int = 250 max_pending_reliable_packets: int | None = None @@ -320,7 +356,7 @@ class CultNetRudpSocketTransportOptions: mode: CultNetRudpSocketMode connection_id: int remote_addr: tuple[str, int] | None = None - initial_sequence: int = 1 + initial_sequence: int | None = None resend_delay_ms: int = 250 transport_id: str = "rudp" max_payload_bytes: int | None = None @@ -409,20 +445,49 @@ class CultNetRudpSession: if options.max_pending_reliable_packets is not None and options.max_pending_reliable_packets <= 0: raise ValueError("RUDP max_pending_reliable_packets must be greater than zero") self.max_pending_reliable_packets = options.max_pending_reliable_packets - self._next_sequence = _uint32(options.initial_sequence, "initial_sequence") + self._next_sequence = _uint32( + _draw_sequence() if options.initial_sequence is None else options.initial_sequence, + "initial_sequence", + ) if self._next_sequence == 0xFFFFFFFF: raise ValueError("RUDP initial_sequence must leave room for a reliable packet") self._next_sequenced_by_channel: dict[str, int] = {} self._next_fragment_id = 1 self._connected = False + # Advances every time a generation ends. Everything issued in a + # generation (writes, flushes) belongs to it and dies with it. + self._generation = 0 + # True from the end of a generation until the next Connect or Accept + # begins one. A flush started in that interval has no live generation + # to wait on. + self._ended = False + # The sequence of the Connect that started the current generation: sent + # by this side, or accepted from the peer. A Connect repeats exactly when + # the session is connected and the Connect carries this sequence. + self._connect_sequence: int | None = None + # A Connect this side sent is unanswered. Only then is an Accept honoured. + self._awaiting_accept = False + # When the unanswered Connect first went out; `due_resends` abandons it + # for a fresh attempt once _RUDP_CONNECT_ATTEMPT_MS has passed. + self._connect_started_at_ms = 0 + # The payload of the Connect this side sent, kept so a fresh attempt can + # always be built whatever became of the pending packet. + self._connect_payload = b"" self._last_received_at_ms: int | None = None self._highest_received_sequence: int | None = None self._received_sequences: set[int] = set() self._latest_sequenced_by_channel: dict[str, int] = {} self._pending_reliable: dict[int, _PendingReliablePacket] = {} self._queued_reliable: deque[CultNetRudpPacket] = deque() - self._ordered_next_sequence_by_channel: dict[str, int] = {} - self._ordered_buffers: dict[str, dict[int, tuple[CultNetRudpDeliveredFrame, int]]] = {} + # Every reliable sequence up to and including this one has been received + # since the peer state was last reset. Only the handshake seeds it: the + # peer's Connect on the accepting side, the peer's Accept on the + # connecting side. Reliable data that arrives before that is refused. + self._received_through: int | None = None + # Ordered frames received but not yet deliverable, keyed by first + # sequence. A frame is held for exactly one reason: a reliable sequence + # below it has not arrived. + self._ordered_held: dict[int, CultNetRudpDeliveredFrame] = {} self._fragment_buffers: dict[tuple[str, int], dict[str, Any]] = {} # Matches cultnet-rs max_pending_fragment_sets. A set that loses one # fragment is never completed; without a bound it is kept for the life @@ -455,23 +520,49 @@ class CultNetRudpSession: def outstanding_reliable_packet_count(self) -> int: return len(self._pending_reliable) + len(self._queued_reliable) - def reset_peer_state(self) -> None: - """Forgets everything learned from the peer; sequence numbers already issued stay issued.""" + @property + def generation(self) -> int: + return self._generation + + @property + def ended(self) -> bool: + return self._ended + + def _end_session(self) -> None: + """The one way a session generation ends. The session stops being + connected and what it still owed the peer dies with it: a write not yet + acknowledged is dropped, so no later session retransmits it or credits + an ack to it. What was learned from the peer is not touched: the peer + may not know the session ended, and forgetting what it sent would let its + retransmits be delivered twice.""" self._connected = False + self._ended = True + self._awaiting_accept = False + self._generation += 1 + self._pending_reliable.clear() + self._queued_reliable.clear() + + def reset_peer_state(self) -> None: + """Ends the current generation and forgets everything learned from the peer; sequence numbers already issued stay issued.""" + self._end_session() self._last_received_at_ms = None self._highest_received_sequence = None self._received_sequences.clear() self._next_sequenced_by_channel.clear() self._latest_sequenced_by_channel.clear() - self._pending_reliable.clear() - self._queued_reliable.clear() - self._ordered_next_sequence_by_channel.clear() - self._ordered_buffers.clear() + self._received_through = None + self._ordered_held.clear() self._fragment_buffers.clear() self._fragment_sets_evicted = 0 def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: + # A session that has had a peer starts a new generation: nothing it + # learned from that peer describes the one this Connect reaches, and + # what it still owed that peer no longer takes room in the queue. + if self._connect_sequence is not None: + self.reset_peer_state() self._ensure_reliable_capacity(1) + self._ended = False packet = self._create_packet( CultNetRudpPacketType.CONNECT, "control", @@ -479,22 +570,75 @@ class CultNetRudpSession: reliable=True, ordered=True, ) + self._connect_sequence = packet.sequence + self._awaiting_accept = True + self._connect_started_at_ms = now_ms + self._connect_payload = packet.payload self._track_reliable(packet, now_ms) return packet + def connect_repeats(self, packet: CultNetRudpPacket) -> bool: + """Whether `packet` is a Connect this session's current generation + already owns: a retransmit of the Connect that started it (the session + is connected and the sequence is that Connect's), or a stale copy of an + earlier attempt by the same client (a sequence before it, within the + receive window). Servers that keep one session per peer ask this to tell + a Connect that starts a new session from one that does not; anything + else that reaches `accept_connect` starts a new generation.""" + return ( + packet.packet_type == CultNetRudpPacketType.CONNECT + and self._connected + and self._connect_sequence is not None + and ( + self._connect_sequence == packet.sequence + or self._connect_is_stale(packet.sequence, self._connect_sequence) + ) + ) + + def _connect_is_stale(self, sequence: int, current: int) -> bool: + """A Connect that precedes the current generation's within the receive + window is the client's earlier attempt, delayed in the network: a client + that retried never sends a lower sequence again. Restarting on it would + strand the client, which honours only the Accept for its newest Connect. + The rule is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): + keep the connection and answer with an Ack. A restarted client whose + random initial sequence lands in this window is answered the same way + and abandons the attempt for a fresh draw (_RUDP_CONNECT_ATTEMPT_MS).""" + return sequence != current and _at_or_before(sequence, current, self.RECEIVED_SEQUENCE_WINDOW) + def accept_connect( self, packet: CultNetRudpPacket, now_ms: int = 0, payload: bytes = b"", ) -> CultNetRudpPacket: + """Answers a Connect. A repeat of the accepted one queues nothing, so a + Connect storm cannot grow the reliable queue: the reply is the Accept + still awaiting acknowledgement, or an Ack once it was acknowledged. A + stale copy of an earlier attempt gets the same reply and changes nothing + else: it is not evidence the peer is alive. Any other Connect ends the + current generation, forgets the peer and accepts a new one.""" self._require_connection(packet) if packet.packet_type != CultNetRudpPacketType.CONNECT: raise ValueError(f"Expected RUDP connect packet, got {packet.packet_type.value}") + if self.connect_repeats(packet): + if self._connect_sequence == packet.sequence: + self._apply_acknowledgements(packet) + self._remember_received(packet.sequence) + self._last_received_at_ms = now_ms + for pending in self._pending_reliable.values(): + if pending.packet.packet_type == CultNetRudpPacketType.ACCEPT: + pending.last_sent_at_ms = now_ms + return pending.packet + return self.create_ack() + self.reset_peer_state() self._ensure_reliable_capacity(1) - self._remember_received(packet.sequence) + self._seed_received(packet.sequence) + self._last_received_at_ms = now_ms + self._connect_sequence = packet.sequence self._connected = True + self._ended = False response = self._create_packet( CultNetRudpPacketType.ACCEPT, "control", @@ -505,25 +649,6 @@ class CultNetRudpSession: self._track_reliable(response, now_ms) return response - def answer_repeated_connect(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpPacket: - """Answers a Connect from a peer this session already accepted. - - The packet's sequence is remembered and nothing is queued, so a Connect - storm cannot grow the reliable queue. The reply is the Accept still awaiting - acknowledgement, or an Ack once it was acknowledged. - """ - self._require_connection(packet) - if packet.packet_type != CultNetRudpPacketType.CONNECT: - raise ValueError(f"Expected RUDP connect packet, got {packet.packet_type.value}") - self._apply_acknowledgements(packet) - self._remember_received(packet.sequence) - self._last_received_at_ms = now_ms - for pending in self._pending_reliable.values(): - if pending.packet.packet_type == CultNetRudpPacketType.ACCEPT: - pending.last_sent_at_ms = now_ms - return pending.packet - return self.create_ack() - def send( self, channel_id: str, @@ -589,17 +714,28 @@ class CultNetRudpSession: def receive(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpReceiveResult: self._require_connection(packet) - self._apply_acknowledgements(packet) + # An Accept counts only while this side's Connect is unanswered and the + # Accept names it. A late or duplicate one, or one from an earlier + # generation, must not seed the watermark or revive an ended session. + honours_accept = ( + packet.packet_type == CultNetRudpPacketType.ACCEPT + and self._awaiting_accept + and self._connect_sequence is not None + and _packet_acknowledges(packet, self._connect_sequence) + ) + if packet.packet_type == CultNetRudpPacketType.ACCEPT and not honours_accept: + return CultNetRudpReceiveResult() + # While this side's Connect awaits its Accept, only the Accept it + # honours retires it: an Ack that names the Connect (a server's reply to + # a repeat or a stale copy) says the server did not start a session. + if honours_accept or not self._awaiting_accept: + self._apply_acknowledgements(packet) ready_to_send = self._promote_queued_reliable(now_ms) self._last_received_at_ms = now_ms - expected_sequence_if_uninitialized = ( - packet.sequence - if self._highest_received_sequence is None - else self._highest_received_sequence + 1 - ) - if packet.packet_type == CultNetRudpPacketType.ACCEPT: - self._remember_received(packet.sequence) + if honours_accept: + self._awaiting_accept = False + self._seed_received(packet.sequence) self._connected = True return CultNetRudpReceiveResult(ready_to_send=ready_to_send) @@ -617,7 +753,7 @@ class CultNetRudpSession: ) if packet.packet_type == CultNetRudpPacketType.DISCONNECT: - self._connected = False + self._end_session() return CultNetRudpReceiveResult( ready_to_send=ready_to_send, disconnected=True, @@ -627,37 +763,38 @@ class CultNetRudpSession: if packet.packet_type != CultNetRudpPacketType.DATA: return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - duplicate = packet.reliable and ( - packet.sequence in self._received_sequences or ( - self._highest_received_sequence is not None - and packet.sequence < self._highest_received_sequence - and self._highest_received_sequence - packet.sequence >= self.RECEIVED_SEQUENCE_WINDOW - ) - ) + # Reliable data before the handshake has seeded the watermark has no place + # in the order: refuse it unremembered, so it is not acknowledged and the + # sender retransmits it once the handshake is done. + if packet.reliable and self._received_through is None: + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) + + duplicate = packet.reliable and self._was_received(packet.sequence) if packet.reliable: self._remember_received(packet.sequence) if duplicate: return CultNetRudpReceiveResult(ready_to_send=ready_to_send) + delivered: list[CultNetRudpDeliveredFrame] = [] reassembled = self._reassemble(packet) - if reassembled is None: - return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - frame, ordered, next_sequence = reassembled - if not ordered and packet.sequenced: - newest_sequence = next_sequence - 1 - latest_sequence = self._latest_sequenced_by_channel.get(frame.channel_id) - if latest_sequence is not None and newest_sequence <= latest_sequence: - return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - self._latest_sequenced_by_channel[frame.channel_id] = newest_sequence - return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) - if not ordered: - return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) - return CultNetRudpReceiveResult( - delivered=tuple( - self._deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized) - ), - ready_to_send=ready_to_send, - ) + if reassembled is not None: + frame, ordered, next_sequence = reassembled + if ordered: + self._ordered_held[frame.sequence] = frame + elif not packet.sequenced: + delivered.append(frame) + else: + newest_sequence = next_sequence - 1 + latest_sequence = self._latest_sequenced_by_channel.get(frame.channel_id) + if latest_sequence is None or newest_sequence > latest_sequence: + self._latest_sequenced_by_channel[frame.channel_id] = newest_sequence + delivered.append(frame) + + # Any reliable packet may have advanced the watermark, a fragment or an + # unordered frame as much as an ordered one, so the drain runs after all + # of them. + delivered.extend(self._drain_ordered()) + return CultNetRudpReceiveResult(delivered=tuple(delivered), ready_to_send=ready_to_send) def create_ack(self) -> CultNetRudpPacket: ack, ack_mask = self._ack_state() @@ -683,7 +820,11 @@ class CultNetRudpSession: ) def create_ack_for_received(self, sequence: int) -> CultNetRudpPacket: + # What `receive` refused is not acknowledged by name (the ack carries only + # what was received), so its sender retransmits it. received_sequence = _uint32(sequence, "received sequence") + if not self._was_received(received_sequence): + return self.create_ack() ack, _ = self._ack_state() if ack >= received_sequence and ack - received_sequence <= 32: return self.create_ack() @@ -693,7 +834,7 @@ class CultNetRudpSession: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) def create_disconnect(self, reason: bytes = b"") -> CultNetRudpPacket: - self._connected = False + self._end_session() return self._create_packet(CultNetRudpPacketType.DISCONNECT, "control", reason) def check_timeout(self, now_ms: int, timeout_ms: int) -> bool: @@ -701,10 +842,13 @@ class CultNetRudpSession: return False if now_ms - self._last_received_at_ms <= timeout_ms: return False - self._connected = False + self._end_session() return True def due_resends(self, now_ms: int) -> tuple[CultNetRudpPacket, ...]: + fresh = self._abandon_unanswered_connect(now_ms) + if fresh is not None: + return (fresh,) due: list[CultNetRudpPacket] = [] for pending in self._pending_reliable.values(): if now_ms - pending.last_sent_at_ms >= self.resend_delay_ms: @@ -712,6 +856,24 @@ class CultNetRudpSession: due.append(pending.packet) return tuple(sorted(due, key=lambda packet: packet.sequence)) + def _abandon_unanswered_connect(self, now_ms: int) -> CultNetRudpPacket | None: + """A Connect unanswered for _RUDP_CONNECT_ATTEMPT_MS is replaced, not + retransmitted further, by a Connect with the same payload and the + abandoned sequence plus the receive window less one. Nothing was ever + sent in an unanswered generation, so no sequence issued so far is owed. + The jump keeps a late copy of the abandoned Connect inside the new one's + stale window, so a server that took the new Connect answers the copy + with an Ack instead of restarting; and it leaves the stale window of + whatever generation the server holds, unless that generation sits + exactly at the jump, and then the next attempt leaves it. The first + Connect of a session is the only one drawn at random.""" + if not self._awaiting_accept or now_ms - self._connect_started_at_ms < _RUDP_CONNECT_ATTEMPT_MS: + return None + if self._connect_sequence is None: + return None + self._next_sequence = min(self._connect_sequence + self.RECEIVED_SEQUENCE_WINDOW - 1, 0xFFFFFFFE) + return self.create_connect(now_ms, self._connect_payload) + def _create_packet( self, packet_type: CultNetRudpPacketType, @@ -806,8 +968,39 @@ class CultNetRudpSession: if packet.ack_mask & (1 << bit): self._pending_reliable.pop(packet.ack - bit - 1, None) + def _seed_received(self, sequence: int) -> None: + """The handshake's one act on the watermark: the peer's Connect or Accept + is the first sequence of the session, whatever else it has sent.""" + self._received_through = sequence + self._remember_received(sequence) + + def _was_received(self, sequence: int) -> bool: + """The one duplicate test: true for a sequence in the window that was + received, for one below the window, and for one at or before the + watermark. The watermark starts at the handshake's seed, so a frame the + peer sent in an earlier generation (every sequence it issued is below the + Connect that began this one) is a duplicate of something already + delivered and is acknowledged, never delivered again.""" + return ( + sequence in self._received_sequences + or ( + self._highest_received_sequence is not None + and sequence < self._highest_received_sequence + and self._highest_received_sequence - sequence >= self.RECEIVED_SEQUENCE_WINDOW + ) + or ( + self._received_through is not None + and _at_or_before(sequence, self._received_through, self.RECEIVED_SEQUENCE_WINDOW) + ) + ) + def _remember_received(self, sequence: int) -> None: self._received_sequences.add(sequence) + if self._received_through is not None: + through = self._received_through + while through + 1 in self._received_sequences: + through += 1 + self._received_through = through if self._highest_received_sequence is None or sequence > self._highest_received_sequence: self._highest_received_sequence = sequence if len(self._received_sequences) > self.RECEIVED_SEQUENCE_WINDOW: @@ -865,58 +1058,15 @@ class CultNetRudpSession: max(sequences) + 1, ) - def _deliver_ordered( - self, - frame: CultNetRudpDeliveredFrame, - next_after_frame: int, - expected_sequence_if_uninitialized: int, - ) -> list[CultNetRudpDeliveredFrame]: - next_sequence = self._ordered_next_sequence_by_channel.get(frame.channel_id) - if next_sequence is None: - next_sequence = min(expected_sequence_if_uninitialized, frame.sequence) - self._ordered_next_sequence_by_channel[frame.channel_id] = next_sequence - while ( - frame.sequence > next_sequence - and next_sequence in self._received_sequences - and next_sequence not in self._ordered_buffers.get(frame.channel_id, {}) - ): - next_sequence += 1 - self._ordered_next_sequence_by_channel[frame.channel_id] = next_sequence - if frame.sequence < next_sequence: + def _drain_ordered(self) -> list[CultNetRudpDeliveredFrame]: + """Delivers, in sequence order, every held ordered frame whose first + sequence is at most one past the watermark. Ordered delivery has one + owner: the watermark. A frame this call does not deliver waits for the + sequence below it, and nothing else releases it.""" + if self._received_through is None: return [] - if frame.sequence > next_sequence: - self._ordered_buffers.setdefault(frame.channel_id, {})[frame.sequence] = (frame, next_after_frame) - return [] - - self._ordered_next_sequence_by_channel[frame.channel_id] = next_after_frame - return [frame, *self._drain_ordered(frame.channel_id)] - - def _drain_ordered(self, channel_id: str) -> list[CultNetRudpDeliveredFrame]: - delivered: list[CultNetRudpDeliveredFrame] = [] - buffer = self._ordered_buffers.get(channel_id) - if buffer is None: - return delivered - - while True: - next_sequence = self._ordered_next_sequence_by_channel[channel_id] - pending = buffer.pop(next_sequence, None) - if pending is None: - break - frame, next_after_frame = pending - delivered.append(frame) - self._ordered_next_sequence_by_channel[channel_id] = next_after_frame - self._skip_received_non_channel_sequences(channel_id) - return delivered - - def _skip_received_non_channel_sequences(self, channel_id: str) -> None: - next_sequence = self._ordered_next_sequence_by_channel.get(channel_id) - while ( - next_sequence is not None - and next_sequence in self._received_sequences - and next_sequence not in self._ordered_buffers.get(channel_id, {}) - ): - next_sequence += 1 - self._ordered_next_sequence_by_channel[channel_id] = next_sequence + releasable = sorted(sequence for sequence in self._ordered_held if sequence <= self._received_through + 1) + return [self._ordered_held.pop(sequence) for sequence in releasable] def _allocate_fragment_id(self) -> int: fragment_id = self._next_fragment_id @@ -1033,6 +1183,10 @@ class CultNetRudpSocketTransportConnection: if packet.connection_id != self.session.connection_id: self._packets_dropped += 1 return None + # A Connect from another endpoint is a new client, whatever sequence it + # carries: the endpoint moves and the Connect starts a new generation. + # Only a Connect from the peer's own endpoint can be its repeat. + connect_from_new_endpoint = False if self.remote_addr is None: # Only a Connect claims the endpoint of a server-mode transport. if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type != CultNetRudpPacketType.CONNECT: @@ -1040,18 +1194,21 @@ class CultNetRudpSocketTransportConnection: return None self.remote_addr = remote_addr elif remote_addr != self.remote_addr: - self._packets_dropped += 1 - return None + if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: + self.remote_addr = remote_addr + connect_from_new_endpoint = True + else: + self._packets_dropped += 1 + return None if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: - # A Connect from the peer this transport already accepted repeats: - # answer it with the Accept already owed and queue nothing. + # The session answers a repeat of the Connect it accepted with the + # Accept already owed, and anything else with a new generation. try: - if self.session.connected: - reply = self.session.answer_repeated_connect(packet, _now_ms()) - else: - reply = self.session.accept_connect(packet, _now_ms()) - self.disconnect_reason = None + if connect_from_new_endpoint: + self.session.reset_peer_state() + reply = self.session.accept_connect(packet, _now_ms()) + self.disconnect_reason = None except ValueError: self._packets_dropped += 1 return None @@ -1105,14 +1262,19 @@ class CultNetRudpSocketTransportConnection: preserved = deque(self._delivered_frames) self._delivered_frames.clear() self.socket.settimeout(poll_timeout) + # A flush belongs to the generation it started in: whatever ends that + # generation, and whatever begins after it, the writes being waited on + # are gone. + generation = self.session.generation try: while True: # An ended session forgot its unacknowledged writes; reporting # them flushed would be a lie. - if self.disconnect_reason is not None: + if self.session.ended or self.session.generation != generation: + reason = self.disconnect_reason raise ConnectionError( "RUDP session ended before its reliable writes were acknowledged: " - + self.disconnect_reason.decode("utf-8", errors="replace") + + ("no reason given" if reason is None else reason.decode("utf-8", errors="replace")) ) if self.session.outstanding_reliable_packet_count == 0: return @@ -1137,7 +1299,10 @@ class CultNetRudpSocketTransportConnection: self._send_packet(self.session.create_ping(payload)) def check_timeout(self, timeout_ms: int, now_ms: int | None = None) -> bool: - return self.session.check_timeout(_now_ms() if now_ms is None else now_ms, timeout_ms) + timed_out = self.session.check_timeout(_now_ms() if now_ms is None else now_ms, timeout_ms) + if timed_out: + self.disconnect_reason = b"session timed out" + return timed_out def poll_resends(self) -> None: for packet in self.session.due_resends(_now_ms()): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 45315e1c..fc5d2cf6 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -30,6 +30,8 @@ from cultnet_py.interop_peer import ( ) from cultnet_py import ( compute_simulation_claim_hash, + create_rudp_schema_transport, + rudp_client_bind_host, CultNetDatabaseChange, CultNetRawClient, CultNetRawDocumentRecord, @@ -94,6 +96,17 @@ from cultnet_py import ( ) +def acknowledges(packet, sequence: int) -> bool: + """Whether the packet's ack field and mask cover the sequence.""" + if packet.ack == sequence: + return True + return ( + packet.ack > sequence + and packet.ack - sequence <= 32 + and (packet.ack_mask & (1 << (packet.ack - sequence - 1))) != 0 + ) + + def bind_udp_socket() -> socket.socket: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(("127.0.0.1", 0)) @@ -616,12 +629,7 @@ class CultNetTests(unittest.TestCase): receiver = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=7, initial_sequence=200, resend_delay_ms=100) ) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 1, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 2, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) first = sender.send( "schema", @@ -638,27 +646,28 @@ class CultNetTests(unittest.TestCase): b"third", CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), ) - self.assertEqual(sender.pending_reliable_sequences, (10, 11, 12)) + self.assertEqual(sender.pending_reliable_sequences, (11, 12, 13)) receiver.receive(first) receiver.receive(third) ack_with_gap = receiver.create_ack() - self.assertEqual(ack_with_gap.ack, 12) - self.assertEqual(ack_with_gap.ack_mask, 0b10 | (1 << 9)) + self.assertEqual(ack_with_gap.ack, 13) + self.assertEqual(ack_with_gap.ack_mask, 0b110) sender.receive(ack_with_gap) - self.assertEqual(sender.pending_reliable_sequences, (11,)) + self.assertEqual(sender.pending_reliable_sequences, (12,)) receiver.receive(second) full_ack = receiver.create_ack() - self.assertEqual(full_ack.ack, 12) - self.assertEqual(full_ack.ack_mask, 0b11 | (1 << 9)) + self.assertEqual(full_ack.ack, 13) + self.assertEqual(full_ack.ack_mask, 0b111) sender.receive(full_ack) self.assertEqual(sender.pending_reliable_sequences, ()) def test_cultnet_rudp_session_schedules_reliable_resends_until_acked(self) -> None: session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=1, resend_delay_ms=100)) session.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 99, 50, 0, 0, "control") + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, ) sent = session.send( "schema", @@ -700,7 +709,7 @@ class CultNetTests(unittest.TestCase): server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=100)) connect = client.create_connect(0, b"join") accept = server.accept_connect(connect, 0) - reply = server.answer_repeated_connect(replace(connect, ack=accept.sequence), 1) + reply = server.accept_connect(replace(connect, ack=accept.sequence), 1) self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) self.assertEqual(server.outstanding_reliable_packet_count, 0) @@ -708,17 +717,23 @@ class CultNetTests(unittest.TestCase): session = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=102, initial_sequence=1, max_pending_reliable_packets=2) ) - session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 102, 50, 0, 0, "control")) + session.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=102, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, + ) session.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) session.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) with self.assertRaisesRegex(ValueError, "reliable send queue is full"): session.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) - self.assertEqual(session.pending_reliable_sequences, (1, 2)) + self.assertEqual(session.pending_reliable_sequences, (2, 3)) fragmented = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=103, initial_sequence=1, max_pending_reliable_packets=3) ) - fragmented.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 103, 50, 0, 0, "control")) + fragmented.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=900)).accept_connect(fragmented.create_connect(0), 0), + 0, + ) with self.assertRaisesRegex(ValueError, "reliable send queue is full"): fragmented.send_many( "schema", @@ -731,12 +746,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0)) first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) @@ -753,8 +763,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_lossy_packets_cannot_create_reliable_ordered_gaps(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) realtime = sender.send("realtime", b"discarded realtime", CultNetRudpSendOptions()) latest = sender.send("latest", b"discarded latest state", CultNetRudpSendOptions(sequenced=True)) @@ -770,8 +779,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=50)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) older = sender.send("latest", b"older", CultNetRudpSendOptions(sequenced=True)) newer = sender.send("latest", b"newer", CultNetRudpSendOptions(sequenced=True)) @@ -786,7 +794,10 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_rejects_unreliable_ordered_delivery(self) -> None: session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=196, initial_sequence=1)) - session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 196, 50, 0, 0, "control")) + session.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=196, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, + ) with self.assertRaises(ValueError) as raised: session.send( @@ -799,12 +810,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_skips_control_packets_while_ordering_schema_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) control = sender.create_ack() @@ -817,12 +823,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_fragments_and_reassembles_reliable_ordered_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) packets = sender.send_many( "schema", @@ -847,8 +848,7 @@ class CultNetTests(unittest.TestCase): connection_id = 457 sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) fragment_count = CultNetRudpSession.RELIABLE_SEND_WINDOW_PACKETS + 17 payload = bytes(index % 251 for index in range(fragment_count * 8)) @@ -1068,14 +1068,13 @@ class CultNetTests(unittest.TestCase): # The peer is told, and the goodbye does not acknowledge the refused frame. (goodbye,) = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.DISCONNECT] self.assertTrue(peer.receive(goodbye, 1).disconnected) - self.assertIn(poison.sequence, peer.pending_reliable_sequences) + self.assertFalse(acknowledges(goodbye, poison.sequence), "the goodbye acknowledged the refused frame") # A retransmit is dropped, not acknowledged. to_server(poison) self.assertIsNone(server.receive_once()) self.assertEqual(server.stats.packets_dropped, 2) self.assertEqual(drain_peer(), []) - self.assertIn(poison.sequence, peer.pending_reliable_sequences) # A Connect claims the endpoint again. again = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) @@ -1145,6 +1144,135 @@ class CultNetTests(unittest.TestCase): peer_socket.close() server.close() + def test_cultnet_rudp_server_mode_admits_a_restarted_client_and_keeps_its_session_on_a_retransmit(self) -> None: + server, _, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x10203058) + connection_id = 0x10203058 + reliable = CultNetRudpSendOptions(reliable=True, ordered=True) + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=50_000)) + connect = first.create_connect(0, b"same") + to_server(connect) + server.receive_once() + first.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + sent = [first.send("schema", f"c{index}".encode(), reliable) for index in range(3)] + delivered = [] + for packet in sent: + to_server(packet) + delivered.append(server.receive_once().payload) + self.assertEqual(delivered, [b"c0", b"c1", b"c2"]) + + # A retransmitted Connect is a repeat: what was delivered stays delivered once. + to_server(connect) + server.receive_once() + to_server(sent[0]) + self.assertIsNone(server.receive_once(), "the retransmitted Connect reset the session") + + # The same client restarted on the same address, connection id and payload. + drain_peer() + second = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=7)) + to_server(second.create_connect(0, b"same")) + server.receive_once() + second.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + to_server(second.send("schema", b"after restart", reliable)) + self.assertEqual(server.receive_once().payload, b"after restart") + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_pinned_client_that_restarts_on_a_new_port(self) -> None: + server, server_socket, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x1020305B) + connection_id = 0x1020305B + reliable = CultNetRudpSendOptions(reliable=True, ordered=True) + other_socket = bind_udp_socket() + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(first.create_connect(0, b"join")) + server.receive_once() + first.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + to_server(first.send("schema", b"hello", reliable)) + self.assertEqual(server.receive_once().payload, b"hello") + + # The process restarted on a new port with the same pinned sequence. + second = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + other_socket.sendto(encode_rudp_packet(second.create_connect(0, b"join")), server_socket.getsockname()) + server.receive_once() + wire, _ = other_socket.recvfrom(65535) + second.receive(decode_rudp_packet(wire), 0) + self.assertTrue(second.connected, "the restarted client on a new port was not admitted") + other_socket.sendto(encode_rudp_packet(second.send("schema", b"after", reliable)), server_socket.getsockname()) + self.assertEqual(server.receive_once().payload, b"after") + finally: + other_socket.close() + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_pinned_client_that_restarts_on_the_same_address(self) -> None: + server, _, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x1020305C) + connection_id = 0x1020305C + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(first.create_connect(0, b"join")) + server.receive_once() + accept = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0] + first.receive(accept, 0) + to_server(first.create_ack_for_received(accept.sequence)) + server.receive_once() + drain_peer() + + # The process restarted on the same address with the same pinned sequence. + restarted = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(restarted.create_connect(0, b"join")) + server.receive_once() + for packet in drain_peer(): + restarted.receive(packet, 1) + self.assertFalse(restarted.connected) + + to_server(restarted.due_resends(3_000)[0]) + server.receive_once() + for packet in drain_peer(): + restarted.receive(packet, 3_001) + self.assertTrue(restarted.connected, "the restarted pinned client was never admitted") + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_connect_from_a_new_endpoint(self) -> None: + server, server_socket, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x10203059) + other_socket = bind_udp_socket() + try: + to_server(peer.create_connect(0, b"a")) + server.receive_once() + drain_peer() + + other = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=0x10203059, initial_sequence=9)) + other_socket.sendto(encode_rudp_packet(other.create_connect(0, b"b")), server_socket.getsockname()) + server.receive_once() + wire, _ = other_socket.recvfrom(65535) + other.receive(decode_rudp_packet(wire), 0) + self.assertTrue(other.connected, "the new endpoint was not admitted") + other_socket.sendto( + encode_rudp_packet(other.send("schema", b"from b", CultNetRudpSendOptions(reliable=True, ordered=True))), + server_socket.getsockname(), + ) + self.assertEqual(server.receive_once().payload, b"from b") + finally: + other_socket.close() + peer_socket.close() + server.close() + + def test_cultnet_rudp_flush_after_a_timeout_names_the_timeout(self) -> None: + server, _, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x1020305A) + try: + to_server(peer.create_connect(0, b"join")) + server.receive_once() + server.send("schema", b"unanswered") + self.assertTrue(server.check_timeout(0, now_ms=10**14)) + with self.assertRaisesRegex(ConnectionError, "session timed out"): + server.flush_reliable(0.1) + finally: + peer_socket.close() + server.close() + def test_cultnet_rudp_session_rejects_limits_that_cannot_admit_a_connect(self) -> None: with self.assertRaises(ValueError): CultNetRudpSession(CultNetRudpSessionOptions(connection_id=1, initial_sequence=0xFFFFFFFF)) @@ -2320,12 +2448,78 @@ class CultNetTests(unittest.TestCase): self.assertEqual(CultNetSimulationConsensusCandidate.from_wire(candidates[0]), typed_candidates[0]) +def _non_loopback_ipv4_address() -> str | None: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.connect(("192.0.2.1", 9)) + address = probe.getsockname()[0] + except OSError: + return None + finally: + probe.close() + return None if address.startswith("127.") or address == "0.0.0.0" else address + + +class CultNetRudpClientBindTests(unittest.TestCase): + def test_an_rudp_client_binds_loopback_only_for_a_loopback_endpoint(self) -> None: + self.assertEqual(rudp_client_bind_host("10.77.0.1"), "0.0.0.0") + self.assertEqual(rudp_client_bind_host("2001:db8::1"), "::") + self.assertEqual(rudp_client_bind_host("[2001:db8::1]"), "::") + self.assertEqual(rudp_client_bind_host("127.0.0.1"), "127.0.0.1") + self.assertEqual(rudp_client_bind_host("127.0.0.2"), "127.0.0.1") + self.assertEqual(rudp_client_bind_host("::1"), "::1") + self.assertEqual(rudp_client_bind_host("[::1]"), "::1") + + # A socket bound to loopback cannot send off the host on Windows. On Linux + # it reaches a local non-loopback address, but from 127.0.0.1, so the + # source address shows which one the client bound. + def test_a_schema_transport_for_a_non_loopback_endpoint_does_not_send_from_loopback(self) -> None: + address = _non_loopback_ipv4_address() + if address is None: + self.skipTest("no non-loopback IPv4 address on this host") + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("0.0.0.0", 0)) + listener.settimeout(2.0) + try: + with self.assertRaises(TimeoutError): + create_rudp_schema_transport( + host=address, + port=listener.getsockname()[1], + connection_id=0x43554C54, + timeout_seconds=0.2, + ) + _, source = listener.recvfrom(65535) + self.assertEqual(source[0], address) + finally: + listener.close() + + def test_a_schema_transport_binds_an_explicit_bind_host(self) -> None: + address = _non_loopback_ipv4_address() + if address is None: + self.skipTest("no non-loopback IPv4 address on this host") + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("0.0.0.0", 0)) + listener.settimeout(2.0) + try: + with self.assertRaises(TimeoutError): + create_rudp_schema_transport( + host=address, + port=listener.getsockname()[1], + connection_id=0x43554C54, + timeout_seconds=0.2, + bind_host="127.0.0.1", + ) + _, source = listener.recvfrom(65535) + self.assertEqual(source[0], "127.0.0.1") + finally: + listener.close() + + class CultNetRudpFragmentBoundTests(unittest.TestCase): def _pair(self) -> tuple[CultNetRudpSession, CultNetRudpSession]: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) return sender, receiver def test_stranded_fragment_sets_are_bounded_and_evicted_oldest_first(self) -> None: diff --git a/packages/cultnet-py/tests/test_rudp_connect_generation.py b/packages/cultnet-py/tests/test_rudp_connect_generation.py new file mode 100644 index 00000000..4bbd896a --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_connect_generation.py @@ -0,0 +1,427 @@ +"""A Connect's sequence says whether it repeats the one a session accepted, and +the handshake alone seeds the watermark that orders delivery.""" +from __future__ import annotations + +import unittest +from dataclasses import replace + +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetRudpSocketMode, + CultNetRudpSocketTransportOptions, +) + +CONNECTION_ID = 0x10203090 + + +def session(initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence)) + + +def send(from_session: CultNetRudpSession, text: str, *, ordered: bool = True, channel: str = "schema"): + return from_session.send(channel, text.encode(), CultNetRudpSendOptions(reliable=True, ordered=ordered)) + + +def names(result) -> list[str]: + return [frame.payload.decode() for frame in result.delivered] + + +def handshake(client: CultNetRudpSession, server: CultNetRudpSession): + connect = client.create_connect(0) + accept = server.accept_connect(connect, 0) + client.receive(accept, 0) + assert client.connected and server.connected + return connect, accept + + +class CultNetRudpConnectGenerationTests(unittest.TestCase): + def test_a_reconnect_on_the_same_session_is_not_stranded_by_a_write_the_ended_session_owed(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + send(server, "lost") + server.create_disconnect(b"gone") + + client.receive(server.accept_connect(client.create_connect(1), 1), 1) + self.assertEqual(names(client.receive(send(server, "after"), 2)), ["after"]) + + def test_a_server_that_accepts_a_new_client_forgets_the_old_clients_sequences(self) -> None: + old_client, server = session(50), session(500) + handshake(old_client, server) + for name in ("c1", "c2", "c3"): + self.assertEqual(names(server.receive(send(old_client, name), 1)), [name]) + server.create_disconnect(b"restart") + + new_client = session(2) + new_client.receive(server.accept_connect(new_client.create_connect(2), 2), 2) + self.assertEqual(names(server.receive(send(new_client, "fresh"), 3)), ["fresh"]) + + def test_a_retransmitted_connect_after_data_has_flowed_does_not_reset_the_session(self) -> None: + client, server = session(1), session(500) + connect, _ = handshake(client, server) + frame = send(client, "once") + self.assertEqual(names(server.receive(frame, 1)), ["once"]) + + self.assertTrue(server.connect_repeats(connect)) + self.assertEqual(server.accept_connect(connect, 2).packet_type, CultNetRudpPacketType.ACK) + self.assertEqual(server.outstanding_reliable_packet_count, 0, "a repeat queued something") + self.assertEqual( + names(server.receive(frame, 3)), [], "a retransmitted Connect made the server forget what it had delivered" + ) + + def test_a_connect_with_another_sequence_is_not_a_repeat_whatever_it_carries(self) -> None: + server = session(500) + connect = session(1).create_connect(0, b"same payload") + server.accept_connect(connect, 0) + self.assertTrue(server.connect_repeats(connect)) + + self.assertFalse(server.connect_repeats(session(9).create_connect(0, b"same payload"))) + self.assertFalse(server.connect_repeats(replace(connect, packet_type=CultNetRudpPacketType.PING))) + server.create_disconnect() + self.assertFalse(server.connect_repeats(connect), "an ended session repeats nothing") + + def test_an_accept_that_does_not_name_the_pending_connect_is_ignored(self) -> None: + client, server = session(10), session(500) + connect = client.create_connect(0) + accept = server.accept_connect(connect, 0) + + client.receive(replace(accept, ack=connect.sequence + 50, ack_mask=0), 1) + self.assertFalse(client.connected, "a stale Accept connected the session") + self.assertEqual(client.pending_reliable_sequences, (connect.sequence,)) + + client.receive(replace(accept, ack=connect.sequence + 3, ack_mask=1 << 2), 2) + self.assertTrue(client.connected, "the mask names the Connect") + + def test_an_accept_after_a_local_disconnect_does_not_reconnect_the_session(self) -> None: + client, server = session(1), session(500) + _, accept = handshake(client, server) + client.create_disconnect(b"bye") + self.assertFalse(client.connected) + + client.receive(accept, 5) + self.assertFalse(client.connected, "a late Accept revived an ended session") + + def test_a_reconnect_forgets_what_the_old_server_sent(self) -> None: + client, old_server = session(1), session(500) + handshake(client, old_server) + for name in ("d1", "d2", "d3"): + self.assertEqual(names(client.receive(send(old_server, name), 1)), [name]) + + new_server = session(501) + client.receive(new_server.accept_connect(client.create_connect(2), 2), 2) + self.assertEqual(names(client.receive(send(new_server, "fresh"), 3)), ["fresh"]) + + def test_an_accept_for_an_abandoned_connect_does_not_connect_the_session(self) -> None: + client, server = session(1), session(500) + accept = server.accept_connect(client.create_connect(0), 0) + client.create_disconnect(b"never mind") + + client.receive(accept, 1) + self.assertFalse(client.connected, "an Accept revived an abandoned Connect") + + def test_a_full_queue_owed_to_a_vanished_peer_does_not_refuse_a_new_generation(self) -> None: + def bounded(initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=CONNECTION_ID, initial_sequence=initial_sequence, max_pending_reliable_packets=1 + ) + ) + + server = bounded(500) + server.accept_connect(session(1).create_connect(0), 0) + self.assertEqual(server.outstanding_reliable_packet_count, 1, "the Accept is owed and never acknowledged") + following = session(9).create_connect(1) + server.accept_connect(following, 1) + self.assertTrue(server.connect_repeats(following)) + + client = bounded(1) + client.create_connect(0) + client.create_connect(1) + self.assertEqual(client.outstanding_reliable_packet_count, 1, "only the new Connect is owed") + + def test_a_duplicate_accept_does_not_reseed_the_watermark(self) -> None: + client, server = session(1), session(500) + _, accept = handshake(client, server) + s1, s2 = send(server, "s1"), send(server, "s2") + client.receive(replace(accept, sequence=s2.sequence), 1) + self.assertEqual(names(client.receive(s2, 2)), [], "s1 is still missing") + self.assertEqual(names(client.receive(s1, 3)), ["s1", "s2"]) + + def test_reliable_data_before_the_accept_is_neither_delivered_nor_acknowledged(self) -> None: + client, server = session(1), session(500) + accept = server.accept_connect(client.create_connect(0), 0) + a, b = send(server, "A"), send(server, "B") + + for early in (b, a): + self.assertEqual(names(client.receive(early, 1)), []) + ack = client.create_ack_for_received(early.sequence) + self.assertEqual((ack.ack, ack.ack_mask), (0, 0), "an unhandled packet was acknowledged") + client.receive(accept, 2) + self.assertTrue(client.connected) + self.assertEqual(server.outstanding_reliable_packet_count, 3, "nothing was acknowledged") + + self.assertEqual(names(client.receive(a, 3)), ["A"]) + self.assertEqual(names(client.receive(b, 3)), ["B"]) + + def test_ordered_frames_after_the_accept_are_delivered_in_order_whatever_arrives_first(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + a, b = send(server, "A"), send(server, "B") + self.assertEqual(names(client.receive(b, 1)), []) + self.assertEqual(names(client.receive(a, 2)), ["A", "B"]) + + def test_an_unordered_reliable_frame_is_delivered_on_receipt_never_held_behind_a_gap(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + send(server, "lost") + unordered = send(server, "now", ordered=False, channel="media") + ordered = send(server, "later") + self.assertEqual(names(client.receive(unordered, 1)), ["now"]) + self.assertEqual(names(client.receive(ordered, 2)), [], "the ordered frame waits for the gap") + + def test_a_peer_disconnect_does_not_forget_what_the_peer_sent(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + s1, s2 = send(client, "s1"), send(client, "s2") + self.assertEqual(names(server.receive(s2, 1)), [], "s1 is missing, so s2 is held") + + server.receive(client.create_disconnect(b"bye"), 2) + self.assertEqual(names(server.receive(s1, 3)), ["s1", "s2"], "the Disconnect forgot the held frame") + self.assertEqual(names(server.receive(s2, 4)), [], "the Disconnect forgot what was received") + + def test_sessions_from_one_options_object_each_draw_their_own_initial_sequence(self) -> None: + self.assertIsNone(CultNetRudpSessionOptions(connection_id=1).initial_sequence) + self.assertIsNone( + CultNetRudpSocketTransportOptions( + runtime_id="r", socket=None, mode=CultNetRudpSocketMode.CLIENT, connection_id=1 + ).initial_sequence + ) + options = CultNetRudpSessionOptions(connection_id=1) + firsts = {CultNetRudpSession(options).create_connect(0).sequence for _ in range(64)} + self.assertGreater(len(firsts), 32, "a default session does not draw its own sequence") + self.assertTrue(all(1 <= value < 2**31 for value in firsts), "a draw left [1, 2^31)") + + # Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed + # Connect from an earlier attempt is stale, a Connect from a new endpoint is + # a new client. + + @staticmethod + def names_sequence(ack, sequence: int) -> bool: + return ack.ack == sequence or any( + ack.ack_mask & (1 << bit) and ack.ack > bit and ack.ack - bit - 1 == sequence for bit in range(32) + ) + + def test_a_frame_from_the_clients_earlier_generation_is_acknowledged_not_delivered_again(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + old = send(client, "old") + self.assertEqual(names(server.receive(old, 1)), ["old"]) + + connect = client.create_connect(2) + client.receive(server.accept_connect(connect, 2), 2) + self.assertLess(old.sequence, connect.sequence) + self.assertEqual(names(server.receive(old, 3)), [], "an earlier generation's frame was delivered again") + self.assertTrue(self.names_sequence(server.create_ack_for_received(old.sequence), old.sequence)) + self.assertEqual(names(server.receive(send(client, "fresh"), 4)), ["fresh"]) + + def test_a_frame_from_the_servers_earlier_generation_is_acknowledged_not_delivered_again(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + old = send(server, "old") + self.assertEqual(names(client.receive(old, 1)), ["old"]) + + client.receive(server.accept_connect(client.create_connect(2), 2), 2) + self.assertEqual(names(client.receive(old, 3)), [], "an earlier generation's frame was delivered again") + self.assertTrue(self.names_sequence(client.create_ack_for_received(old.sequence), old.sequence)) + self.assertEqual(names(client.receive(send(server, "fresh"), 4)), ["fresh"]) + + def test_a_duplicate_below_the_receive_window_is_acknowledged_by_name(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + first = send(client, "first") + self.assertEqual(names(server.receive(first, 1)), ["first"]) + for _ in range(4_200): + server.receive(send(client, "x"), 1) + client.receive(server.create_ack(), 1) + self.assertEqual(names(server.receive(first, 2)), []) + self.assertEqual( + server.create_ack_for_received(first.sequence).ack, + first.sequence, + "a duplicate below the window was not acknowledged by name", + ) + + def test_a_delayed_connect_from_an_earlier_attempt_does_not_strand_the_client(self) -> None: + client, server = session(10), session(500) + earlier = client.create_connect(0) + retried = client.create_connect(300) + accept = server.accept_connect(retried, 301) + client.receive(accept, 302) + server.receive(client.create_ack_for_received(accept.sequence), 302) + self.assertTrue(client.connected) + a = send(client, "a") + self.assertEqual(names(server.receive(a, 303)), ["a"]) + client.receive(server.create_ack_for_received(a.sequence), 303) + + self.assertTrue(server.connect_repeats(earlier), "the delayed Connect starts nothing") + reply = server.accept_connect(earlier, 304) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + client.receive(reply, 305) + self.assertTrue(client.connected) + + b = send(client, "b") + self.assertEqual(names(server.receive(b, 306)), ["b"], "the delayed Connect reset the server") + client.receive(server.create_ack_for_received(b.sequence), 307) + self.assertNotIn(b.sequence, client.pending_reliable_sequences) + + def test_a_repeated_connect_refreshes_liveness_and_a_stale_one_does_not(self) -> None: + client, server = session(10), session(500) + earlier = client.create_connect(0) + current = client.create_connect(1) + server.accept_connect(current, 0) + + server.accept_connect(current, 900) + self.assertFalse(server.check_timeout(1_000, 500), "a repeated Connect did not refresh liveness") + + server.accept_connect(earlier, 1_400) + self.assertTrue(server.check_timeout(1_600, 500), "a stale Connect refreshed liveness") + + def test_stale_connects_are_recognised_by_serial_arithmetic(self) -> None: + server = session(500) + server.accept_connect(session(3).create_connect(0), 0) + + def connect(sequence: int): + return session(sequence).create_connect(0) + + self.assertTrue(server.connect_repeats(connect(2**32 - 2)), "just before the wrap") + self.assertFalse(server.connect_repeats(connect(4)), "just after") + self.assertFalse(server.connect_repeats(connect(3 + 4_096)), "far after") + self.assertTrue(server.connect_repeats(connect(2**32 - 4_092)), "the window's edge") + self.assertFalse(server.connect_repeats(connect(2**32 - 4_093)), "past the window") + + def test_a_restarted_client_whose_first_sequence_is_stale_connects_after_redrawing(self) -> None: + server, old = session(500), session(1_000) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + + restarted = session(900) + first = restarted.create_connect(0, b"join") + # The old client has not acknowledged its Accept yet, so that is the + # reply: it names the old Connect, not this one. + early = server.accept_connect(first, 1) + self.assertEqual(early.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(early, 1) + self.assertFalse(restarted.connected, "an Accept for another Connect connected the client") + server.receive(old.create_ack_for_received(old_accept.sequence), 1) + reply = server.accept_connect(first, 2) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + restarted.receive(reply, 2) + self.assertFalse(restarted.connected) + + retransmitted = restarted.due_resends(1_000) + self.assertEqual(len(retransmitted), 1) + self.assertEqual(retransmitted[0].sequence, first.sequence, "the attempt is still young") + + fresh = restarted.due_resends(3_500) + self.assertEqual(len(fresh), 1) + self.assertEqual(fresh[0].packet_type, CultNetRudpPacketType.CONNECT) + self.assertNotEqual(fresh[0].sequence, first.sequence, "the same Connect was retransmitted for ever") + self.assertEqual(fresh[0].payload, b"join", "the fresh attempt lost the Connect payload") + + accept = server.accept_connect(fresh[0], 3_500) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 3_500) + self.assertTrue(restarted.connected) + self.assertEqual(names(server.receive(send(restarted, "hello"), 3_501)), ["hello"]) + + def test_an_answered_connect_is_never_replaced_by_the_attempt_timeout(self) -> None: + client, server = session(10), session(500) + connect, _ = handshake(client, server) + resent = client.due_resends(60_000) + self.assertTrue( + all(p.packet_type != CultNetRudpPacketType.CONNECT or p.sequence == connect.sequence for p in resent), + "a connected client started a new attempt", + ) + self.assertTrue(client.connected) + + def test_an_accepted_peer_that_goes_silent_times_out(self) -> None: + server = session(500) + server.accept_connect(session(10).create_connect(0), 0) + self.assertTrue(server.check_timeout(100_000, 1_000)) + + + # Ack Cut 1d, batch 2: the connect-attempt timeout. + + def test_an_ack_naming_the_pending_connect_does_not_retire_it(self) -> None: + server, old = session(500), session(1) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + server.receive(old.create_ack_for_received(old_accept.sequence), 0) + + restarted = session(1) + first = restarted.create_connect(0, b"join") + reply = server.accept_connect(first, 1) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + self.assertTrue(self.names_sequence(reply, first.sequence), "the Ack must name the pending Connect for this test to bite") + restarted.receive(reply, 1) + self.assertFalse(restarted.connected) + self.assertEqual(restarted.pending_reliable_sequences, (first.sequence,), "an Ack retired the Connect") + + fresh = restarted.due_resends(3_000) + self.assertEqual(len(fresh), 1, "the client waits for ever with nothing to resend") + self.assertEqual(fresh[0].payload, b"join") + accept = server.accept_connect(fresh[0], 3_000) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 3_000) + self.assertTrue(restarted.connected) + + def test_a_late_copy_of_an_abandoned_connect_is_stale_once_its_replacement_is_accepted(self) -> None: + server, client = session(500), session(10) + abandoned = client.create_connect(0) + fresh = client.due_resends(3_000) + self.assertEqual(len(fresh), 1) + self.assertEqual(fresh[0].sequence, abandoned.sequence + 4_095) + accept = server.accept_connect(fresh[0], 3_001) + client.receive(accept, 3_002) + server.receive(client.create_ack_for_received(accept.sequence), 3_002) + self.assertTrue(client.connected) + a = send(client, "a") + self.assertEqual(names(server.receive(a, 3_003)), ["a"]) + client.receive(server.create_ack_for_received(a.sequence), 3_003) + + self.assertTrue(server.connect_repeats(abandoned), "the late copy would restart the server") + reply = server.accept_connect(abandoned, 3_004) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + client.receive(reply, 3_005) + self.assertTrue(client.connected) + self.assertEqual(names(server.receive(send(client, "b"), 3_006)), ["b"]) + + def test_a_server_sitting_at_the_jump_is_left_by_the_next_attempt(self) -> None: + server, old = session(500), session(5_095) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + server.receive(old.create_ack_for_received(old_accept.sequence), 0) + + restarted = session(1_000) + first = restarted.create_connect(0, b"join") + restarted.receive(server.accept_connect(first, 1), 1) + self.assertFalse(restarted.connected) + + second = restarted.due_resends(3_000)[0] + self.assertEqual(second.sequence, 5_095, "the jump lands on the server's generation") + reply = server.accept_connect(second, 3_001) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + restarted.receive(reply, 3_001) + self.assertFalse(restarted.connected) + + third = restarted.due_resends(6_000)[0] + self.assertEqual(third.sequence, 5_095 + 4_095) + accept = server.accept_connect(third, 6_001) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 6_001) + self.assertTrue(restarted.connected) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/tests/test_rudp_ordered_delivery.py b/packages/cultnet-py/tests/test_rudp_ordered_delivery.py new file mode 100644 index 00000000..0417aeb0 --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_ordered_delivery.py @@ -0,0 +1,115 @@ +"""Ordered delivery has one owner: the contiguous received-through watermark. + +A frame is held for exactly one reason, a reliable sequence below it has not +arrived, and it is delivered in the call that fills the last such gap, +whichever channel the gap belonged to. +""" +from __future__ import annotations + +import unittest + +from cultnet_py.transport import ( + CultNetRudpPacket, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, +) + + +def handshake() -> tuple[CultNetRudpSession, CultNetRudpSession]: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=500)) + accept = server.accept_connect(client.create_connect(0), 0) + client.receive(accept, 0) + return client, server + + +def send(session: CultNetRudpSession, channel: str, payload: str, ordered: bool) -> CultNetRudpPacket: + return session.send(channel, payload.encode(), CultNetRudpSendOptions(reliable=True, ordered=ordered)) + + +def names(result) -> list[str]: + return [frame.payload.decode() for frame in result.delivered] + + +class CultNetRudpOrderedDeliveryTests(unittest.TestCase): + def test_ordered_frame_waits_for_a_gap_filled_by_another_channel(self) -> None: + sender, receiver = handshake() + o1 = send(sender, "schema", "o1", True) + u = send(sender, "rel", "u", False) + o2 = send(sender, "schema", "o2", True) + o3 = send(sender, "schema", "o3", True) + o4 = send(sender, "schema", "o4", True) + + self.assertEqual(names(receiver.receive(o1, 1)), ["o1"]) + self.assertEqual(names(receiver.receive(o2, 2)), [], "u is missing") + self.assertEqual(names(receiver.receive(o3, 3)), [], "u is missing") + self.assertEqual(names(receiver.receive(o4, 4)), [], "u is missing") + self.assertEqual(names(receiver.receive(u, 5)), ["u", "o2", "o3", "o4"]) + + def test_two_ordered_channels_release_each_other_in_sequence_order(self) -> None: + sender, receiver = handshake() + a1 = send(sender, "schema", "A1", True) + b1 = send(sender, "other", "B1", True) + a2 = send(sender, "schema", "A2", True) + a3 = send(sender, "schema", "A3", True) + + delivered = names(receiver.receive(a1, 1)) + names(receiver.receive(a2, 2)) + names(receiver.receive(a3, 3)) + self.assertEqual(delivered, ["A1"], "B1 is missing, so A2 and A3 wait") + self.assertEqual(names(receiver.receive(b1, 4)), ["B1", "A2", "A3"]) + + def test_channel_first_used_after_other_traffic_loses_nothing(self) -> None: + sender, receiver = handshake() + c1 = send(sender, "late", "C1", True) + c2 = send(sender, "late", "C2", True) + x = send(sender, "schema", "X", True) + + self.assertEqual(names(receiver.receive(x, 1)), [], "C1 is missing") + self.assertEqual(names(receiver.receive(c2, 2)), [], "C1 is missing") + self.assertEqual(names(receiver.receive(c1, 3)), ["C1", "C2", "X"]) + + def test_accept_seeds_the_watermark_of_the_connecting_side(self) -> None: + client, server = handshake() + s1 = send(server, "schema", "s1", True) + s2 = send(server, "schema", "s2", True) + + self.assertEqual(names(client.receive(s2, 1)), []) + self.assertEqual(names(client.receive(s1, 2)), ["s1", "s2"]) + + def test_connect_seeds_the_watermark_of_the_accepting_side(self) -> None: + client, server = handshake() + first = send(client, "schema", "first", True) + second = send(client, "schema", "second", True) + + self.assertEqual(names(server.receive(second, 1)), []) + self.assertEqual(names(server.receive(first, 2)), ["first", "second"]) + + def test_duplicate_of_a_held_frame_is_not_delivered_twice(self) -> None: + sender, receiver = handshake() + s1 = send(sender, "schema", "s1", True) + s2 = send(sender, "schema", "s2", True) + + self.assertEqual(names(receiver.receive(s2, 1)), []) + self.assertEqual(names(receiver.receive(s2, 2)), []) + self.assertEqual(names(receiver.receive(s1, 3)), ["s1", "s2"]) + self.assertEqual(names(receiver.receive(s2, 4)), []) + + def test_reset_peer_state_forgets_held_frames_and_reseeds_the_watermark_from_the_next_handshake(self) -> None: + sender, receiver = handshake() + send(sender, "schema", "s1", True) + s2 = send(sender, "schema", "s2", True) + self.assertEqual(names(receiver.receive(s2, 1)), []) + + receiver.reset_peer_state() + following = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=100)) + following.receive(receiver.accept_connect(following.create_connect(0), 2), 2) + d1 = send(following, "schema", "d1", True) + d2 = send(following, "schema", "d2", True) + + # The held s2 died with the reset, and the watermark follows the new peer. + self.assertEqual(names(receiver.receive(d2, 3)), []) + self.assertEqual(names(receiver.receive(d1, 4)), ["d1", "d2"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/tests/test_rudp_session_ending.py b/packages/cultnet-py/tests/test_rudp_session_ending.py new file mode 100644 index 00000000..6e042b1c --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_session_ending.py @@ -0,0 +1,223 @@ +"""A write belongs to the session generation it was issued in. + +Every way a generation ends drops the writes it still owed, and a flush +belongs to the generation it started in. +""" +from __future__ import annotations + +import socket +import threading +import time +import unittest + +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetRudpSocketMode, + CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, + decode_rudp_packet, + encode_rudp_packet, +) + +CONNECTION_ID = 0x10203070 + + +def bind_udp_socket() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.02) + return sock + + +def client_with_a_lost_write() -> tuple[CultNetRudpSession, CultNetRudpSession]: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + client.receive(server.accept_connect(client.create_connect(0), 0), 0) + client.send("schema", b"owed to the old session", CultNetRudpSendOptions(reliable=True, ordered=True)) + # A fragmented write larger than the send window leaves part of it queued. + client.send_many( + "schema", bytes(40 * 8), CultNetRudpSendOptions(reliable=True, ordered=True), max_fragment_bytes=8 + ) + assert client.outstanding_reliable_packet_count == 41 + assert client.queued_reliable_packet_count > 0 + return client, server + + +def drain(sock: socket.socket) -> list: + packets = [] + while True: + try: + wire, _ = sock.recvfrom(65535) + except TimeoutError: + return packets + packets.append(decode_rudp_packet(wire)) + + +def connected_client() -> tuple[CultNetRudpSocketTransportConnection, socket.socket, tuple]: + peer_socket = bind_udp_socket() + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-client", + socket=bind_udp_socket(), + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=peer_socket.getsockname(), + connection_id=CONNECTION_ID, + ) + ) + client.connect(b"hello") + wire, client_addr = peer_socket.recvfrom(65535) + peer = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + accept = peer.accept_connect(decode_rudp_packet(wire), 0) + peer_socket.sendto(encode_rudp_packet(accept), client_addr) + client.receive_once() + assert client.connected + return client, peer_socket, client_addr + + +class CultNetRudpSessionEndingTests(unittest.TestCase): + def test_every_way_a_session_ends_drops_the_writes_it_owed(self) -> None: + endings = { + "peer Disconnect": lambda client, server: client.receive(server.create_disconnect(b"bye"), 1), + "local disconnect": lambda client, server: client.create_disconnect(b"bye"), + "timeout": lambda client, server: self.assertTrue(client.check_timeout(1_000, 10)), + "reset": lambda client, server: client.reset_peer_state(), + } + for name, end in endings.items(): + with self.subTest(ending=name): + client, server = client_with_a_lost_write() + end(client, server) + self.assertEqual(client.outstanding_reliable_packet_count, 0, "the write survived the end") + + next_server = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=900) + ) + next_server.accept_connect(client.create_connect(2_000), 2_000) + for resend in client.due_resends(60_000): + self.assertEqual( + next_server.receive(resend, 60_000).delivered, + (), + "the old session's write was delivered in the next session", + ) + + def test_a_flush_fails_after_a_local_disconnect_or_a_timeout(self) -> None: + client, peer_socket, _ = connected_client() + try: + client.send("schema", b"owed") + client.disconnect(b"bye") + with self.assertRaises(ConnectionError): + client.flush_reliable(0.2) + finally: + client.close() + peer_socket.close() + + client, peer_socket, _ = connected_client() + try: + client.send("schema", b"owed") + time.sleep(0.005) + self.assertTrue(client.check_timeout(1)) + with self.assertRaises(ConnectionError): + client.flush_reliable(0.2) + finally: + client.close() + peer_socket.close() + + def test_a_flush_that_started_before_the_end_fails_even_after_a_later_connect(self) -> None: + client, peer_socket, _ = connected_client() + outcome: list[BaseException | None] = [] + + def flush() -> None: + try: + client.flush_reliable(5.0) + outcome.append(None) + except BaseException as error: # noqa: BLE001 - the outcome is the assertion + outcome.append(error) + + try: + client.send("schema", b"owed") + worker = threading.Thread(target=flush) + worker.start() + time.sleep(0.05) + client.disconnect(b"bye") + client.connect(b"hello again") + worker.join(10) + self.assertEqual(len(outcome), 1) + self.assertIsInstance(outcome[0], ConnectionError) + finally: + client.close() + peer_socket.close() + + + def test_a_flush_started_after_a_reconnect_waits_on_the_new_session(self) -> None: + client, peer_socket, client_addr = connected_client() + try: + client.send("schema", b"owed to the old session") + client.disconnect(b"bye") + client.connect(b"hello again") + connect = [p for p in drain(peer_socket) if p.packet_type == CultNetRudpPacketType.CONNECT][-1] + accept = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=700) + ).accept_connect(connect, 0) + peer_socket.sendto(encode_rudp_packet(accept), client_addr) + client.receive_once() + + client.flush_reliable(0.5) + finally: + client.close() + peer_socket.close() + + def test_a_flush_started_after_the_server_accepts_a_new_connect_waits_on_the_new_session(self) -> None: + server_socket = bind_udp_socket() + peer_socket = bind_udp_socket() + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=CONNECTION_ID, + ) + ) + to_server = lambda packet: peer_socket.sendto(encode_rudp_packet(packet), server_socket.getsockname()) + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID)) + to_server(first.create_connect(0)) + server.receive_once() + to_server(first.create_disconnect()) + server.receive_once() + + following = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID)) + to_server(following.create_connect(0)) + server.receive_once() + accept = [p for p in drain(peer_socket) if p.packet_type == CultNetRudpPacketType.ACCEPT][-1] + following.receive(accept, 0) + to_server(following.create_ack()) + server.receive_once() + + server.flush_reliable(0.5) + finally: + server.close() + peer_socket.close() + + + def test_an_ending_does_not_forget_what_was_received_from_the_peer(self) -> None: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + client.receive(server.accept_connect(client.create_connect(0), 0), 0) + ordered = CultNetRudpSendOptions(reliable=True, ordered=True) + s1 = client.send("schema", b"s1", ordered) + s2 = client.send("schema", b"s2", ordered) + self.assertEqual(server.receive(s2, 1).delivered, (), "s1 is missing, so s2 is held") + + self.assertTrue(server.check_timeout(1_000, 10)) + self.assertEqual( + [frame.payload for frame in server.receive(s1, 1_001).delivered], + [b"s1", b"s2"], + "the ending forgot the held frame", + ) + self.assertEqual(server.receive(s2, 1_002).delivered, (), "the ending forgot what was received") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-rs/Cargo.toml b/packages/cultnet-rs/Cargo.toml index f289d86e..b1cd8d11 100644 --- a/packages/cultnet-rs/Cargo.toml +++ b/packages/cultnet-rs/Cargo.toml @@ -19,6 +19,7 @@ cultcache-rs = { path = "../cultcache-rs" } ed25519-dalek = "2.2" hmac = "0.12" rand = "0.9" +reed-solomon-erasure = "6" rmp-serde = "1.3" rmpv = { version = "1.3", features = ["with-serde"] } serde = { version = "1", features = ["derive"] } @@ -35,3 +36,8 @@ windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Securit [dev-dependencies] pretty_assertions = "1.4" tempfile = "3" + +[[example]] +name = "cultnet_impair" +path = "examples/cultnet_impair/main.rs" +test = true diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index 15880fa1..59764338 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -26,9 +26,9 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended.", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/examples/cultnet_impair/main.rs b/packages/cultnet-rs/examples/cultnet_impair/main.rs new file mode 100644 index 00000000..c2ebd874 --- /dev/null +++ b/packages/cultnet-rs/examples/cultnet_impair/main.rs @@ -0,0 +1,336 @@ +//! `cultnet-impair`: a deterministic bidirectional UDP proxy for pressure-testing +//! real CultNet RUDP sessions. It owns transport impairment only; what a stream +//! does with the loss is the stream's business. +//! +//! The proxy listens at the endpoint a client dials and forwards to `--upstream` +//! through one upstream socket per client, so the reverse ACK and feedback +//! traffic crosses the same schedule. The schedule is `policy.rs`. +//! +//! ```text +//! cargo run --release --example cultnet_impair -- \ +//! --listen 0.0.0.0:17890 --upstream 127.0.0.1:17990 \ +//! --profile tests/impairment/loss-1pct.toml --seed 424242 --metrics loss-1pct.csv +//! ``` +//! +//! Profiles are a small TOML subset of unsigned integers; see +//! `tests/impairment/`. The receiver binds the upstream endpoint and the client +//! dials the proxy's; never one port for both. For a two-host test, run the proxy +//! on the receiver host so reverse traffic is impaired too. + +mod policy; + +use std::collections::HashMap; +use std::env; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::net::{SocketAddr, UdpSocket}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, Instant}; + +use policy::{Direction, Profile, Scheduler, Stats}; + +const MAX_DATAGRAM: usize = 65_535; +const MAX_CLIENT_FLOWS: usize = 64; +const CLIENT_FLOW_IDLE_TIMEOUT_MS: u64 = 30_000; +const MAX_DATAGRAMS_PER_FLOW_PER_TURN: usize = 16; + +#[derive(Debug)] +struct UpstreamFlow { + socket: UdpSocket, + last_activity_ms: u64, +} + +fn open_upstream_flow(upstream: SocketAddr, now_ms: u64) -> io::Result { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.connect(upstream)?; + socket.set_nonblocking(true)?; + Ok(UpstreamFlow { + socket, + last_activity_ms: now_ms, + }) +} + +fn ensure_upstream_flow( + flows: &mut HashMap, + client: SocketAddr, + upstream: SocketAddr, + now_ms: u64, + stats: &mut Stats, +) -> io::Result<()> { + if flows.contains_key(&client) { + return Ok(()); + } + if flows.len() >= MAX_CLIENT_FLOWS + && let Some(oldest) = flows + .iter() + .min_by_key(|(_, flow)| flow.last_activity_ms) + .map(|(address, _)| *address) + { + flows.remove(&oldest); + stats.flow_evicted += 1; + } + flows.insert(client, open_upstream_flow(upstream, now_ms)?); + stats.max_flows = stats.max_flows.max(flows.len()); + Ok(()) +} + +fn reap_idle_flows( + flows: &mut HashMap, + now_ms: u64, + stats: &mut Stats, +) { + let before = flows.len(); + flows.retain(|_, flow| { + now_ms.saturating_sub(flow.last_activity_ms) < CLIENT_FLOW_IDLE_TIMEOUT_MS + }); + stats.flow_evicted += (before - flows.len()) as u64; +} + +#[derive(Debug)] +struct Options { + listen: SocketAddr, + upstream: SocketAddr, + profile: PathBuf, + seed: u64, + metrics: Option, +} + +fn parse_options() -> Result { + let mut args = env::args().skip(1); + let mut listen = None; + let mut upstream = None; + let mut profile = None; + let mut seed = 1; + let mut metrics = None; + while let Some(arg) = args.next() { + let mut value = || args.next().ok_or_else(|| format!("{arg} requires a value")); + match arg.as_str() { + "--listen" => { + listen = Some( + value()? + .parse() + .map_err(|_| "--listen must be a socket address")?, + ) + } + "--upstream" => { + upstream = Some( + value()? + .parse() + .map_err(|_| "--upstream must be a socket address")?, + ) + } + "--profile" => profile = Some(PathBuf::from(value()?)), + "--seed" => { + seed = value()? + .parse() + .map_err(|_| "--seed must be an unsigned integer")? + } + "--metrics" => metrics = Some(PathBuf::from(value()?)), + "--help" | "-h" => return Err(usage().into()), + _ => return Err(format!("unknown argument {arg}\n{}", usage())), + } + } + Ok(Options { + listen: listen.ok_or("--listen is required")?, + upstream: upstream.ok_or("--upstream is required")?, + profile: profile.ok_or("--profile is required")?, + seed, + metrics, + }) +} + +fn usage() -> &'static str { + "usage: cultnet-impair --listen HOST:PORT --upstream HOST:PORT --profile PATH [--seed N] [--metrics PATH]" +} + +fn elapsed_ms(start: Instant) -> u64 { + start.elapsed().as_millis().try_into().unwrap_or(u64::MAX) +} + +fn write_metrics(path: &Path, stats: &Stats, active_flows: usize) -> io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = File::create(path)?; + writeln!( + file, + "received,forwarded,dropped,duplicated,reordered,stalled,queue_overflow,max_queue,flow_evicted,max_flows,active_flows" + )?; + writeln!( + file, + "{},{},{},{},{},{},{},{},{},{},{}", + stats.received, + stats.forwarded, + stats.dropped, + stats.duplicated, + stats.reordered, + stats.stalled, + stats.queue_overflow, + stats.max_queue, + stats.flow_evicted, + stats.max_flows, + active_flows + ) +} + +fn run(options: Options) -> Result<(), String> { + let profile = Profile::load(&options.profile)?; + let client_socket = UdpSocket::bind(options.listen) + .map_err(|error| format!("binding {}: {error}", options.listen))?; + client_socket + .set_nonblocking(true) + .map_err(|error| error.to_string())?; + println!( + "cultnet-impair listen={} upstream={} seed={} profile={}", + options.listen, + options.upstream, + options.seed, + options.profile.display() + ); + let start = Instant::now(); + let mut scheduler = Scheduler::new(profile, options.seed); + let mut flows = HashMap::::new(); + let mut last_flow_reap_at = 0_u64; + let mut buffer = vec![0; MAX_DATAGRAM]; + loop { + let now = elapsed_ms(start); + let mut did_work = false; + for _ in 0..MAX_DATAGRAMS_PER_FLOW_PER_TURN { + match client_socket.recv_from(&mut buffer) { + Ok((size, address)) => { + did_work = true; + ensure_upstream_flow( + &mut flows, + address, + options.upstream, + now, + &mut scheduler.stats, + ) + .map_err(|error| format!("opening upstream flow for {address}: {error}"))?; + if let Some(flow) = flows.get_mut(&address) { + flow.last_activity_ms = now; + } + scheduler.admit( + now, + Direction::ClientToUpstream, + &buffer[..size], + address, + ); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => break, + Err(error) => return Err(error.to_string()), + } + } + for (address, flow) in &mut flows { + for _ in 0..MAX_DATAGRAMS_PER_FLOW_PER_TURN { + match flow.socket.recv(&mut buffer) { + Ok(size) => { + did_work = true; + flow.last_activity_ms = now; + scheduler.admit( + now, + Direction::UpstreamToClient, + &buffer[..size], + *address, + ); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => break, + Err(error) => return Err(error.to_string()), + } + } + } + while let Some(item) = scheduler.pop_due(now) { + did_work = true; + let result = match item.direction { + Direction::ClientToUpstream => match flows.get_mut(&item.client) { + Some(flow) => { + flow.last_activity_ms = now; + flow.socket.send(&item.bytes) + } + None => { + scheduler.stats.dropped += 1; + continue; + } + }, + Direction::UpstreamToClient => client_socket.send_to(&item.bytes, item.client), + }; + if result.is_ok() { + scheduler.stats.forwarded += 1; + } + } + if let Some(path) = options.metrics.as_deref() { + if now % 1000 == 0 { + let _ = write_metrics(path, &scheduler.stats, flows.len()); + } + } + if now.saturating_sub(last_flow_reap_at) >= 1_000 { + reap_idle_flows(&mut flows, now, &mut scheduler.stats); + last_flow_reap_at = now; + } + if did_work { + thread::yield_now(); + } else { + thread::sleep(Duration::from_millis(1)); + } + } +} + +fn main() { + match parse_options().and_then(run) { + Ok(()) => {} + Err(error) => { + eprintln!("{error}"); + std::process::exit(2); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn independent_clients_keep_independent_upstream_udp_identities() { + let mut flows = HashMap::new(); + let mut stats = Stats::default(); + let upstream = "127.0.0.1:54321".parse().unwrap(); + let first = "127.0.0.1:12001".parse().unwrap(); + let second = "127.0.0.1:12002".parse().unwrap(); + + ensure_upstream_flow(&mut flows, first, upstream, 1, &mut stats).unwrap(); + ensure_upstream_flow(&mut flows, second, upstream, 2, &mut stats).unwrap(); + + assert_eq!(flows.len(), 2); + assert_ne!( + flows[&first].socket.local_addr().unwrap(), + flows[&second].socket.local_addr().unwrap() + ); + assert_eq!(stats.max_flows, 2); + assert_eq!(stats.flow_evicted, 0); + } + + #[test] + fn idle_client_flows_are_reaped_without_touching_active_flows() { + let mut flows = HashMap::new(); + let mut stats = Stats::default(); + let upstream = "127.0.0.1:54321".parse().unwrap(); + let idle = "127.0.0.1:12001".parse().unwrap(); + let active = "127.0.0.1:12002".parse().unwrap(); + + ensure_upstream_flow(&mut flows, idle, upstream, 0, &mut stats).unwrap(); + ensure_upstream_flow( + &mut flows, + active, + upstream, + CLIENT_FLOW_IDLE_TIMEOUT_MS - 1, + &mut stats, + ) + .unwrap(); + reap_idle_flows(&mut flows, CLIENT_FLOW_IDLE_TIMEOUT_MS, &mut stats); + + assert!(!flows.contains_key(&idle)); + assert!(flows.contains_key(&active)); + assert_eq!(stats.flow_evicted, 1); + } +} diff --git a/packages/cultnet-rs/examples/cultnet_impair/policy.rs b/packages/cultnet-rs/examples/cultnet_impair/policy.rs new file mode 100644 index 00000000..844052cf --- /dev/null +++ b/packages/cultnet-rs/examples/cultnet_impair/policy.rs @@ -0,0 +1,298 @@ +//! The impairment policy: what happens to each datagram, and nothing about +//! sockets. `Profile` says what to do, `Scheduler` decides per datagram and +//! holds the delayed ones, and every decision is a pure function of the +//! profile, the seed and the datagram's position, so a schedule replays +//! exactly. The proxy in `main.rs` and the loss-matrix test in +//! `tests/media_fec_loss.rs` both take their schedule from here. + +use std::collections::VecDeque; +use std::fs; +use std::net::SocketAddr; +use std::path::Path; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Profile { + pub loss_basis_points: u32, + pub burst_every: u64, + pub burst_length: u64, + pub duplicate_every: u64, + pub reorder_every: u64, + pub reorder_delay_ms: u64, + pub delay_ms: u64, + pub jitter_ms: u64, + pub stall_at_ms: u64, + pub stall_for_ms: u64, + pub max_scheduled: usize, +} + +impl Default for Profile { + fn default() -> Self { + Self { + loss_basis_points: 0, + burst_every: 0, + burst_length: 0, + duplicate_every: 0, + reorder_every: 0, + reorder_delay_ms: 0, + delay_ms: 0, + jitter_ms: 0, + stall_at_ms: 0, + stall_for_ms: 0, + max_scheduled: 4096, + } + } +} + +impl Profile { + pub fn load(path: &Path) -> Result { + let text = fs::read_to_string(path) + .map_err(|error| format!("reading {}: {error}", path.display()))?; + let mut profile = Self::default(); + for (index, raw) in text.lines().enumerate() { + let line = raw.split('#').next().unwrap_or("").trim(); + if line.is_empty() || line.starts_with('[') { + continue; + } + let (key, value) = line + .split_once('=') + .ok_or_else(|| format!("{}:{}: expected key = value", path.display(), index + 1))?; + let key = key.trim(); + let value = value.trim().trim_matches('"'); + let number = value.parse::().map_err(|_| { + format!( + "{}:{}: {key} must be an unsigned integer", + path.display(), + index + 1 + ) + })?; + match key { + "loss_basis_points" => { + profile.loss_basis_points = number + .try_into() + .map_err(|_| "loss_basis_points is too large")? + } + "burst_every" => profile.burst_every = number, + "burst_length" => profile.burst_length = number, + "duplicate_every" => profile.duplicate_every = number, + "reorder_every" => profile.reorder_every = number, + "reorder_delay_ms" => profile.reorder_delay_ms = number, + "delay_ms" => profile.delay_ms = number, + "jitter_ms" => profile.jitter_ms = number, + "stall_at_ms" => profile.stall_at_ms = number, + "stall_for_ms" => profile.stall_for_ms = number, + "max_scheduled" => { + profile.max_scheduled = number + .try_into() + .map_err(|_| "max_scheduled is too large")? + } + _ => { + return Err(format!( + "{}:{}: unknown profile key {key}", + path.display(), + index + 1 + )); + } + } + } + if profile.loss_basis_points > 10_000 { + return Err("loss_basis_points must be <= 10000".into()); + } + if profile.max_scheduled == 0 { + return Err("max_scheduled must be greater than zero".into()); + } + Ok(profile) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Direction { + ClientToUpstream, + UpstreamToClient, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Decision { + pub drop: bool, + pub copies: u8, + pub delay_ms: u64, + pub reordered: bool, + pub stalled: bool, +} + +#[derive(Clone, Debug)] +pub struct DeterministicRng(u64); + +impl DeterministicRng { + pub fn new(seed: u64) -> Self { + Self(if seed == 0 { 0x9e3779b97f4a7c15 } else { seed }) + } + pub fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } +} + +#[derive(Debug)] +pub struct Scheduler { + pub profile: Profile, + pub rng: DeterministicRng, + pub seen: u64, + pub burst_remaining: u64, + pub queue: VecDeque, + pub stats: Stats, +} + +#[derive(Debug)] +pub struct Scheduled { + pub due_ms: u64, + pub direction: Direction, + pub bytes: Vec, + pub client: SocketAddr, +} + +#[derive(Debug, Default)] +pub struct Stats { + pub received: u64, + pub forwarded: u64, + pub dropped: u64, + pub duplicated: u64, + pub reordered: u64, + pub stalled: u64, + pub queue_overflow: u64, + pub max_queue: usize, + pub flow_evicted: u64, + pub max_flows: usize, +} + +impl Scheduler { + pub fn new(profile: Profile, seed: u64) -> Self { + Self { + profile, + rng: DeterministicRng::new(seed), + seen: 0, + burst_remaining: 0, + queue: VecDeque::new(), + stats: Stats::default(), + } + } + + pub fn decide(&mut self, now_ms: u64) -> Decision { + self.seen += 1; + let stalled = self.profile.stall_for_ms > 0 + && now_ms >= self.profile.stall_at_ms + && now_ms + < self + .profile + .stall_at_ms + .saturating_add(self.profile.stall_for_ms); + if stalled { + return Decision { + drop: true, + copies: 0, + delay_ms: 0, + reordered: false, + stalled: true, + }; + } + if self.burst_remaining == 0 + && self.profile.burst_every > 0 + && self.seen % self.profile.burst_every == 0 + { + self.burst_remaining = self.profile.burst_length; + } + if self.burst_remaining > 0 { + self.burst_remaining -= 1; + return Decision { + drop: true, + copies: 0, + delay_ms: 0, + reordered: false, + stalled: false, + }; + } + if self.profile.loss_basis_points > 0 + && self.rng.next() % 10_000 < u64::from(self.profile.loss_basis_points) + { + return Decision { + drop: true, + copies: 0, + delay_ms: 0, + reordered: false, + stalled: false, + }; + } + let duplicate = + self.profile.duplicate_every > 0 && self.seen % self.profile.duplicate_every == 0; + let reordered = + self.profile.reorder_every > 0 && self.seen % self.profile.reorder_every == 0; + let jitter = if self.profile.jitter_ms == 0 { + 0 + } else { + self.rng.next() % (self.profile.jitter_ms + 1) + }; + Decision { + drop: false, + copies: if duplicate { 2 } else { 1 }, + delay_ms: self.profile.delay_ms + + jitter + + if reordered { + self.profile.reorder_delay_ms + } else { + 0 + }, + reordered, + stalled: false, + } + } + + pub fn admit(&mut self, now_ms: u64, direction: Direction, bytes: &[u8], client: SocketAddr) { + self.stats.received += 1; + let decision = self.decide(now_ms); + if decision.drop { + self.stats.dropped += 1; + if decision.stalled { + self.stats.stalled += 1; + } + return; + } + if decision.copies == 2 { + self.stats.duplicated += 1; + } + if decision.reordered { + self.stats.reordered += 1; + } + for copy in 0..decision.copies { + if self.queue.len() >= self.profile.max_scheduled { + self.stats.queue_overflow += 1; + self.stats.dropped += 1; + continue; + } + let delay = decision.delay_ms + u64::from(copy); + let item = Scheduled { + due_ms: now_ms.saturating_add(delay), + direction, + bytes: bytes.to_vec(), + client, + }; + let position = self + .queue + .iter() + .position(|queued| queued.due_ms > item.due_ms) + .unwrap_or(self.queue.len()); + self.queue.insert(position, item); + self.stats.max_queue = self.stats.max_queue.max(self.queue.len()); + } + } + + pub fn pop_due(&mut self, now_ms: u64) -> Option { + if self.queue.front().is_some_and(|item| item.due_ms <= now_ms) { + self.queue.pop_front() + } else { + None + } + } +} diff --git a/packages/cultnet-rs/examples/cultnet_interop_peer.rs b/packages/cultnet-rs/examples/cultnet_interop_peer.rs index 2d8a1678..3d6700bf 100644 --- a/packages/cultnet-rs/examples/cultnet_interop_peer.rs +++ b/packages/cultnet-rs/examples/cultnet_interop_peer.rs @@ -507,7 +507,7 @@ fn rudp_serve_once(options: &BTreeMap) -> Result<()> { let local_port = socket.local_addr()?.port(); let mut transport_options = CultNetRudpSocketTransportOptions::server("rust-rudp-interop", socket, 0x22446688); - transport_options.initial_sequence = 100; + transport_options.initial_sequence = Some(100); transport_options.resend_delay_ms = 25; transport_options.max_fragment_bytes = max_fragment_bytes; let mut transport = CultNetRudpSocketTransportConnection::new(transport_options)?; @@ -608,7 +608,7 @@ fn rudp_serve_message_once(options: &BTreeMap) -> Result<()> { let local_port = socket.local_addr()?.port(); let mut transport_options = CultNetRudpSocketTransportOptions::server("rust-rudp-message-interop", socket, 0x22446689); - transport_options.initial_sequence = 100; + transport_options.initial_sequence = Some(100); transport_options.resend_delay_ms = 25; let mut transport = CultNetRudpSocketTransportConnection::new(transport_options)?; @@ -967,7 +967,7 @@ fn run_rudp_server( let peer = peers.entry(remote_addr).or_insert_with(|| RudpPeerSession { session: CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: RUDP_INTEROP_CONNECTION_ID, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: RUDP_INTEROP_RESEND_DELAY_MS, max_pending_reliable_packets: None, }), diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index f31ade85..1808c62f 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -250,8 +250,8 @@ impl CultNetShardDescriptor { } } -/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the five strings -/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a fifth +/// Machine-readable `cultnet.error.v0` refusal code (R-N). Mirrors the strings +/// `contracts/cultnet/cultnet.error.schema.json` enumerates for `code`; a peer that receives a /// string it does not know refuses to decode rather than guess a meaning for it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -261,6 +261,7 @@ pub enum CultNetErrorCode { CursorInvalid, ReferenceOutsideTarget, VariantUnsupported, + UnownedSchema, } impl CultNetErrorCode { @@ -271,6 +272,7 @@ impl CultNetErrorCode { Self::CursorInvalid => "cursor_invalid", Self::ReferenceOutsideTarget => "reference_outside_target", Self::VariantUnsupported => "variant_unsupported", + Self::UnownedSchema => "unowned_schema", } } @@ -281,6 +283,7 @@ impl CultNetErrorCode { "cursor_invalid" => Ok(Self::CursorInvalid), "reference_outside_target" => Ok(Self::ReferenceOutsideTarget), "variant_unsupported" => Ok(Self::VariantUnsupported), + "unowned_schema" => Ok(Self::UnownedSchema), other => Err(anyhow!("ErrorMessage.Code {other:?} is not a recognized cultnet.error.v0 code")), } } diff --git a/packages/cultnet-rs/src/cultmesh.rs b/packages/cultnet-rs/src/cultmesh.rs index 14998396..920a9f1f 100644 --- a/packages/cultnet-rs/src/cultmesh.rs +++ b/packages/cultnet-rs/src/cultmesh.rs @@ -3,6 +3,9 @@ use anyhow::anyhow; use chrono::DateTime; use chrono::Utc; use std::collections::BTreeMap; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::net::SocketAddr; use std::net::ToSocketAddrs; use std::net::UdpSocket; @@ -240,10 +243,13 @@ impl CultNetRudpEndpoint { #[derive(Clone, Debug)] pub struct CultMeshRudpSocketOptions { - pub bind_host: String, + /// Local address to bind. Unset, a server binds loopback, and a client binds + /// loopback for a loopback endpoint and the unspecified address of the + /// endpoint's family otherwise, so it can reach a remote host. + pub bind_host: Option, pub bind_port: u16, pub read_timeout: Option, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub max_payload_bytes: Option, pub max_fragment_bytes: Option, @@ -253,10 +259,10 @@ pub struct CultMeshRudpSocketOptions { impl Default for CultMeshRudpSocketOptions { fn default() -> Self { Self { - bind_host: "127.0.0.1".to_string(), + bind_host: None, bind_port: 0, read_timeout: Some(Duration::from_millis(20)), - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, max_payload_bytes: None, max_fragment_bytes: None, @@ -316,7 +322,8 @@ impl CultMesh { connection_id: u32, options: CultMeshRudpSocketOptions, ) -> Result { - let socket = bind_rudp_socket(&options)?; + let bind_host = options.bind_host.as_deref().unwrap_or("127.0.0.1"); + let socket = bind_rudp_socket(bind_host, &options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_reliable_expire_after_ms: None, media_delivery: None, @@ -341,14 +348,19 @@ impl CultMesh { endpoint: &CultNetRudpEndpoint, options: CultMeshRudpSocketOptions, ) -> Result { - let socket = bind_rudp_socket(&options)?; + let remote_addr = endpoint.socket_addr()?; + let bind_host = match options.bind_host.as_deref() { + Some(host) => host.to_string(), + None => client_bind_ip(remote_addr).to_string(), + }; + let socket = bind_rudp_socket(&bind_host, &options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_reliable_expire_after_ms: None, media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Client, - remote_addr: Some(endpoint.socket_addr()?), + remote_addr: Some(remote_addr), connection_id, initial_sequence: options.initial_sequence, resend_delay_ms: options.resend_delay_ms, @@ -503,10 +515,21 @@ pub fn cultnet_rudp_endpoint(endpoint: &str) -> Result { Ok(CultNetRudpEndpoint { host, port }) } -fn bind_rudp_socket(options: &CultMeshRudpSocketOptions) -> Result { +/// A socket bound to loopback cannot send off the host on Windows, so a client +/// keeps loopback only when its peer is on loopback. +fn client_bind_ip(remote_addr: SocketAddr) -> IpAddr { + match (remote_addr.ip().is_loopback(), remote_addr) { + (true, SocketAddr::V4(_)) => IpAddr::V4(Ipv4Addr::LOCALHOST), + (true, SocketAddr::V6(_)) => IpAddr::V6(Ipv6Addr::LOCALHOST), + (false, SocketAddr::V4(_)) => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + (false, SocketAddr::V6(_)) => IpAddr::V6(Ipv6Addr::UNSPECIFIED), + } +} + +fn bind_rudp_socket(bind_host: &str, options: &CultMeshRudpSocketOptions) -> Result { let socket = UdpSocket::bind(format!( "{}:{}", - socket_host(&options.bind_host), + socket_host(bind_host), options.bind_port ))?; socket.set_read_timeout(options.read_timeout)?; diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index 5bfd1857..621150ba 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -2,6 +2,7 @@ mod content; mod contracts; mod idunn_deployment_brake; mod idunn_lifecycle_brake; +mod media_fec; mod media_stream_contracts; mod media_stream_wire; mod provider_health_contracts; @@ -24,6 +25,7 @@ pub use content::*; pub use contracts::*; pub use idunn_deployment_brake::*; pub use idunn_lifecycle_brake::*; +pub use media_fec::*; pub use media_stream_contracts::*; pub use media_stream_wire::*; pub use provider_health_contracts::*; diff --git a/packages/cultnet-rs/src/media_fec.rs b/packages/cultnet-rs/src/media_fec.rs new file mode 100644 index 00000000..490eaaed --- /dev/null +++ b/packages/cultnet-rs/src/media_fec.rs @@ -0,0 +1,749 @@ +//! Forward error correction for CultMesh Media Stream records. +//! +//! Pure functions over the records in [`crate::media_stream_contracts`]: no +//! sockets, no clock, no randomness. A producer calls [`protect_video_frame`] +//! and [`protect_audio_block`]; a consumer calls [`recover_video_block`] and +//! [`recover_audio_block`]. Both ends link this one implementation, so a +//! producer and a consumer cannot disagree about the math without one of them +//! failing to build against the same version of this crate. +//! +//! # The code +//! +//! Every block is a systematic Reed-Solomon code over GF(2^8): `k` data shards +//! are sent unchanged and `m` parity shards are computed from them. Any `k` of +//! the `k + m` shards recover the block (the code is MDS), so any `m` erasures +//! are repaired and any more are reported as [`MediaFecError::BeyondRepair`], +//! never as wrong bytes. The construction is named on every parity record by +//! [`MEDIA_FEC_SCHEME_RS_GF256_V1`] and backed by the `reed-solomon-erasure` +//! crate. The library is not the contract: the construction below is, and +//! `tests/media_fec.rs` pins the parity bytes of known inputs, so a library that +//! changes the matrix fails there instead of on the wire, and a decoder in +//! another runtime has vectors to meet. +//! +//! # The `rs-gf256-v1` construction +//! +//! Enough to implement it in another runtime without this crate. +//! +//! * The field is GF(2^8) over the polynomial `x^8 + x^4 + x^3 + x^2 + 1` +//! (`0x11D`). Addition is XOR; multiplication is carry-less multiplication +//! reduced by `0x11D`. +//! * For a block of `k` data shards and `m` parity shards, let `n = k + m` +//! (`n <= 256`) and build the `n x k` Vandermonde matrix `V` whose entry in +//! row `r`, column `c` is `r^c`, where the row number `r` (0 through +//! `n - 1`) is itself read as a field element and `0^0 = 1`. The evaluation +//! points are the integers 0 through `n - 1`, not powers of a generator. +//! * The encoding matrix is `E = V * inverse(top)`, where `top` is the first +//! `k` rows of `V`. Its first `k` rows are the identity, so data shards are +//! sent unchanged; its last `m` rows generate parity. +//! * Every shard of a block has the same length, `shard_payload_bytes`. A +//! shorter final video chunk is zero-padded to that length before encoding. +//! Byte `j` of parity shard `p` is the XOR over data shards `i` of +//! `E[k + p][i] * data[i][j]`. +//! * Any `k` surviving shards recover the block: invert the `k x k` submatrix +//! of `E` for the surviving rows and multiply. +//! +//! The known-answer vectors in `tests/media_fec.rs` were checked against an +//! independent implementation of exactly this text. +//! +//! # What a block is +//! +//! * Video: a frame's chunks split into `ceil(n / 16)` contiguous, near-equal +//! blocks, each protected by `max(2, ceil(k / 4))` parity shards. A block +//! never spans two frames, so one frame's deadline governs its recovery. +//! [`protect_video_frame`] returns the frame's records in send order: blocks +//! are interleaved, so a burst of consecutive losses lands as one loss per +//! block rather than many in one. +//! * Audio: `4` consecutive packets of one payload length, protected by `2` +//! parity shards. +//! +//! # One datagram +//! +//! Every record the protect functions return, data or parity, fits one +//! datagram once wrapped for the wire, so parity never reintroduces the +//! fragment amplification it exists to remove. The protect functions enforce +//! that against [`MediaFecPolicy::max_wire_bytes`]: they encode each record +//! with the caller's provenance and refuse the block if any is larger. +//! The default budget is [`MEDIA_FEC_MAX_WIRE_BYTES`], an IPv4 path with a +//! 1500-byte MTU. Pass a smaller budget when the path carries less: IPv6 over +//! 1500 leaves a UDP payload of 1452 bytes, and a WireGuard or Tailscale +//! tunnel about 1252, so subtract the 41 bytes of RUDP header and channel name +//! from the path's UDP payload limit. +//! +//! The budget is measured with the provenance given, so a producer whose +//! `stored_at` or identifiers grow later must pass the widest it will use. +//! +//! # Audio playout +//! +//! A recovered audio packet has the block's latest `deadline_ticks`, so a +//! packet recovered early in the block reports a deadline later than its own. +//! Audio consumers must gate playout on `pts_ticks`, which recovery derives +//! exactly, and use `deadline_ticks` only to give up on the block as a whole. +//! +//! # What this module does not own +//! +//! When to give up on a block is the receiver's deadline, and the `(k, m)` +//! choice is the producer's policy ([`MediaFecPolicy`]); this module only makes +//! and consumes shards. + +use std::collections::BTreeMap; +use std::fmt; + +use reed_solomon_erasure::galois_8::ReedSolomon; + +use crate::media_stream_contracts::{ + GameCultMediaAudioPacketRecord, GameCultMediaAudioParityShardRecord, + GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, +}; +use crate::media_stream_wire::{ + GAMECULT_MEDIA_CHANNEL, GAMECULT_MEDIA_MAX_WIRE_BYTES, GameCultMediaWireRecord, MediaWireProvenance, + encode_media_wire_record, validate_audio_parity_record, validate_audio_record, + validate_video_parity_record, validate_video_record, +}; +use crate::rudp::RUDP_FIXED_HEADER_BYTES; + +/// Names the construction on every parity record: systematic Reed-Solomon over +/// GF(2^8), as built by `reed-solomon-erasure`'s `galois_8` field. A different +/// matrix is a different scheme id. +pub const MEDIA_FEC_SCHEME_RS_GF256_V1: &str = "rs-gf256-v1"; + +/// The largest UDP payload an IPv4 datagram carries on a 1500-byte MTU. +pub const MEDIA_IPV4_UDP_PAYLOAD_BYTES: usize = 1_472; + +/// The default datagram budget: the most a wrapped media record may occupy so +/// that the RUDP header and the channel name still leave it in one IPv4 +/// datagram on a 1500-byte MTU (1,431 bytes). +pub const MEDIA_FEC_MAX_WIRE_BYTES: usize = + MEDIA_IPV4_UDP_PAYLOAD_BYTES - RUDP_FIXED_HEADER_BYTES - GAMECULT_MEDIA_CHANNEL.len(); + +/// A producer's choice of `(k, m)` and datagram budget. +/// [`MediaFecPolicy::STANDARD`] is the ruled default; each record carries the +/// `(k, m)` it was made with, so a decoder needs no policy. +/// +/// Build one by overriding fields of the default, `MediaFecPolicy { +/// max_wire_bytes: 1_211, ..MediaFecPolicy::STANDARD }`, not by naming every +/// field: a field added later then cannot break, or silently mis-set, the +/// caller. `max_wire_bytes` may not exceed +/// [`GAMECULT_MEDIA_MAX_WIRE_BYTES`], the ceiling a decoder holds every record +/// to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MediaFecPolicy { + /// The most data chunks in one video block. + pub video_max_block_data_shards: u16, + /// The fewest parity shards a video block carries. + pub video_min_parity_shards: u16, + /// A video block of `k` data shards carries at least `ceil(k / this)` parity + /// shards. + pub video_parity_divisor: u16, + /// Packets per audio block. + pub audio_data_shards: u16, + /// Parity shards per audio block. + pub audio_parity_shards: u16, + /// The most a wrapped record may occupy. The protect functions refuse a + /// block with any record over it. See the module docs for smaller paths. + pub max_wire_bytes: usize, +} + +impl MediaFecPolicy { + /// Video `k <= 16` with `m = max(2, ceil(k / 4))`; audio 4+2; the IPv4 + /// datagram budget. + pub const STANDARD: Self = Self { + video_max_block_data_shards: 16, + video_min_parity_shards: 2, + video_parity_divisor: 4, + audio_data_shards: 4, + audio_parity_shards: 2, + max_wire_bytes: MEDIA_FEC_MAX_WIRE_BYTES, + }; + + /// `m` for a video block of `data_shards` chunks. + pub fn video_parity_shards(&self, data_shards: u16) -> u16 { + data_shards + .div_ceil(self.video_parity_divisor.max(1)) + .max(self.video_min_parity_shards) + } + + /// How a frame of `chunk_count` chunks splits: `ceil(n / max)` blocks whose + /// sizes differ by at most one, larger blocks first. + pub fn video_block_sizes(&self, chunk_count: u16) -> Vec { + let max = self.video_max_block_data_shards.max(1); + let blocks = chunk_count.div_ceil(max); + if blocks == 0 { + return Vec::new(); + } + let base = chunk_count / blocks; + let extra = chunk_count % blocks; + (0..blocks).map(|index| base + u16::from(index < extra)).collect() + } + + fn validate(&self) -> Result<(), MediaFecError> { + let widest = self.video_max_block_data_shards; + if widest == 0 + || self.video_min_parity_shards == 0 + || self.video_parity_divisor == 0 + || self.audio_data_shards == 0 + || self.audio_parity_shards == 0 + { + return Err(MediaFecError::invalid("media FEC policy fields must be non-zero")); + } + if usize::from(widest) + usize::from(self.video_parity_shards(widest)) > 256 + || usize::from(self.audio_data_shards) + usize::from(self.audio_parity_shards) > 256 + { + return Err(MediaFecError::invalid("media FEC policy exceeds 256 shards per block")); + } + if self.max_wire_bytes > GAMECULT_MEDIA_MAX_WIRE_BYTES { + return Err(MediaFecError::invalid(format!( + "media FEC policy max_wire_bytes {} exceeds the {GAMECULT_MEDIA_MAX_WIRE_BYTES}-byte media record ceiling", + self.max_wire_bytes + ))); + } + Ok(()) + } +} + +impl Default for MediaFecPolicy { + fn default() -> Self { + Self::STANDARD + } +} + +/// Why a block could not be protected or recovered. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MediaFecError { + /// Fewer than `k` of the block's shards are present. Nothing is returned: + /// a block that cannot be repaired is reported, never guessed at. + BeyondRepair { shards_present: usize, shards_needed: usize }, + /// The inputs are not one well-formed block. + Invalid(String), +} + +impl MediaFecError { + fn invalid(message: impl Into) -> Self { + Self::Invalid(message.into()) + } +} + +impl fmt::Display for MediaFecError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BeyondRepair { shards_present, shards_needed } => write!( + f, + "media FEC block is beyond repair: {shards_present} shards present, {shards_needed} needed" + ), + Self::Invalid(message) => write!(f, "invalid media FEC block: {message}"), + } + } +} + +impl std::error::Error for MediaFecError {} + +// --------------------------------------------------------------------------- +// The code: equal-length shards in, equal-length shards out. +// --------------------------------------------------------------------------- + +fn parity_shards(data: &[Vec], parity_count: usize) -> Result>, MediaFecError> { + let code = ReedSolomon::new(data.len(), parity_count) + .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; + let mut parity = vec![vec![0_u8; data[0].len()]; parity_count]; + code.encode_sep(data, &mut parity) + .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; + Ok(parity) +} + +/// `shards` holds the block's `k` data slots then `m` parity slots, `None` for +/// each missing. Returns the `k` data shards. +fn recover_data_shards( + data_shards: usize, + mut shards: Vec>>, +) -> Result>, MediaFecError> { + let parity_shards = shards.len() - data_shards; + let present = shards.iter().flatten().count(); + if present < data_shards { + return Err(MediaFecError::BeyondRepair { + shards_present: present, + shards_needed: data_shards, + }); + } + let code = ReedSolomon::new(data_shards, parity_shards) + .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; + code.reconstruct_data(&mut shards) + .map_err(|error| MediaFecError::invalid(format!("{error:?}")))?; + Ok(shards + .into_iter() + .take(data_shards) + .map(|shard| shard.expect("reconstruct_data fills every data shard")) + .collect()) +} + +fn padded(payload: &[u8], shard_bytes: usize) -> Vec { + let mut shard = vec![0_u8; shard_bytes]; + shard[..payload.len()].copy_from_slice(payload); + shard +} + +fn from_anyhow(error: anyhow::Error) -> MediaFecError { + MediaFecError::Invalid(error.to_string()) +} + +/// Refuses a record that would not fit one datagram once wrapped. +fn fits_datagram( + record: &GameCultMediaWireRecord, + provenance: MediaWireProvenance<'_>, + policy: &MediaFecPolicy, +) -> Result<(), MediaFecError> { + let wire = encode_media_wire_record(record, provenance).map_err(from_anyhow)?; + if wire.len() > policy.max_wire_bytes { + return Err(MediaFecError::invalid(format!( + "a {} record wraps to {} bytes, over the {}-byte datagram budget: cut chunks smaller or raise max_wire_bytes", + record.schema_id(), + wire.len(), + policy.max_wire_bytes + ))); + } + Ok(()) +} + +/// Where slot `slot` of an audio block lands: its packet id and presentation +/// time, or `None` if either overflows. The one place block arithmetic lives; +/// the protect path, the wire validator and recovery all ask it. +pub(crate) fn audio_slot( + base_packet_id: u64, + base_pts_ticks: i64, + packet_duration_ticks: u32, + slot: u64, +) -> Option<(u64, i64)> { + let packet_id = base_packet_id.checked_add(slot)?; + let pts_ticks = i64::from(packet_duration_ticks) + .checked_mul(i64::try_from(slot).ok()?)? + .checked_add(base_pts_ticks)?; + Some((packet_id, pts_ticks)) +} + +// --------------------------------------------------------------------------- +// Video +// --------------------------------------------------------------------------- + +/// Protects one video frame. `chunks` are all of the frame's access-unit +/// records, in any order; the result is every data and parity record in send +/// order. +/// +/// Within a block, data and parity alternate; across blocks, the sequence +/// rotates one record from each block at a time. Where every block is the same +/// size, no two consecutive `block_count` records share a block. Blocks that +/// differ in size run out at the end, and the rotation then cycles over the +/// blocks remaining. +/// +/// Errors if `chunks` are not exactly one whole frame (mixed frames, a missing +/// or repeated chunk index, disagreeing framing), if a chunk is not shard +/// shaped (every chunk of a block but its last must be the same length, the +/// last no longer), or if any resulting record, wrapped with `provenance`, +/// exceeds `policy.max_wire_bytes`. +pub fn protect_video_frame( + chunks: &[GameCultMediaVideoAccessUnitRecord], + policy: &MediaFecPolicy, + provenance: MediaWireProvenance<'_>, +) -> Result, MediaFecError> { + policy.validate()?; + let first = chunks + .first() + .ok_or_else(|| MediaFecError::invalid("a video frame needs at least one chunk"))?; + let chunk_count = usize::from(first.chunk_count); + if chunks.len() != chunk_count { + return Err(MediaFecError::invalid(format!( + "frame {} declares {} chunks but {} were given", + first.frame_id, + chunk_count, + chunks.len() + ))); + } + let mut by_index: BTreeMap = BTreeMap::new(); + for chunk in chunks { + validate_video_record(chunk).map_err(from_anyhow)?; + if !same_frame(first, chunk) { + return Err(MediaFecError::invalid( + "a block never spans frames: chunks disagree on stream, session, frame or framing", + )); + } + if by_index.insert(chunk.chunk_index, chunk).is_some() { + return Err(MediaFecError::invalid(format!( + "chunk {} appears twice", + chunk.chunk_index + ))); + } + } + + let block_sizes = policy.video_block_sizes(first.chunk_count); + let block_count = block_sizes.len() as u16; + let mut lanes: Vec> = Vec::with_capacity(block_sizes.len()); + let mut start = 0_u16; + for (block_index, &size) in block_sizes.iter().enumerate() { + let members: Vec<&GameCultMediaVideoAccessUnitRecord> = (start..start + size) + .map(|index| by_index[&index]) + .collect(); + let shard_bytes = members[0].payload.len(); + let (last, body) = members.split_last().expect("block has a member"); + if body.iter().any(|chunk| chunk.payload.len() != shard_bytes) + || last.payload.len() > shard_bytes + { + return Err(MediaFecError::invalid( + "every chunk of a block but its last must share one length, and the last no longer", + )); + } + let data: Vec> = members + .iter() + .map(|chunk| padded(&chunk.payload, shard_bytes)) + .collect(); + let parity_count = policy.video_parity_shards(size); + let parity = parity_shards(&data, usize::from(parity_count))?; + + let parity_records: Vec = parity + .into_iter() + .enumerate() + .map(|(parity_index, payload)| { + GameCultMediaWireRecord::VideoParity(GameCultMediaVideoParityShardRecord { + stream_id: first.stream_id.clone(), + session_id: first.session_id.clone(), + frame_id: first.frame_id, + codec: first.codec.clone(), + pts_ticks: first.pts_ticks, + duration_ticks: first.duration_ticks, + timebase_num: first.timebase_num, + timebase_den: first.timebase_den, + keyframe: first.keyframe, + dependency_frame_id: first.dependency_frame_id, + deadline_ticks: first.deadline_ticks, + chunk_count: first.chunk_count, + fec_scheme: MEDIA_FEC_SCHEME_RS_GF256_V1.to_string(), + block_index: block_index as u16, + block_count, + block_data_start: start, + block_data_count: size, + parity_index: parity_index as u16, + parity_count, + shard_payload_bytes: shard_bytes as u32, + last_chunk_payload_bytes: last.payload.len() as u32, + payload, + }) + }) + .collect(); + let data_records = members + .iter() + .map(|chunk| GameCultMediaWireRecord::Video((*chunk).clone())); + lanes.push(interleave(data_records.collect(), parity_records)); + start += size; + } + let records = rotate(lanes); + for record in &records { + fits_datagram(record, provenance, policy)?; + } + Ok(records) +} + +fn same_frame(a: &GameCultMediaVideoAccessUnitRecord, b: &GameCultMediaVideoAccessUnitRecord) -> bool { + a.stream_id == b.stream_id + && a.session_id == b.session_id + && a.frame_id == b.frame_id + && a.codec == b.codec + && a.pts_ticks == b.pts_ticks + && a.duration_ticks == b.duration_ticks + && a.timebase_num == b.timebase_num + && a.timebase_den == b.timebase_den + && a.keyframe == b.keyframe + && a.dependency_frame_id == b.dependency_frame_id + && a.deadline_ticks == b.deadline_ticks + && a.chunk_count == b.chunk_count +} + +/// Data and parity alternate: `D0 P0 D1 P1 ...`, the longer list finishing the +/// lane. +fn interleave(data: Vec, parity: Vec) -> Vec { + let mut data = data.into_iter(); + let mut parity = parity.into_iter(); + let mut lane = Vec::new(); + loop { + let next_data = data.next(); + let next_parity = parity.next(); + if next_data.is_none() && next_parity.is_none() { + return lane; + } + lane.extend(next_data); + lane.extend(next_parity); + } +} + +/// One record from each lane in turn, until every lane is empty. +fn rotate(lanes: Vec>) -> Vec { + let mut lanes: Vec<_> = lanes.into_iter().map(Vec::into_iter).collect(); + let mut out = Vec::new(); + loop { + let before = out.len(); + for lane in &mut lanes { + out.extend(lane.next()); + } + if out.len() == before { + return out; + } + } +} + +/// Rebuilds the data chunks a block lost. +/// +/// `parity` are the block's parity shards that arrived (at least one, for its +/// geometry) and `data` its data chunks that arrived; each may be in any order +/// and may repeat. Returns the missing chunks, ordered by `chunk_index`: empty +/// if none were missing. Fails with [`MediaFecError::BeyondRepair`] if fewer +/// than `block_data_count` shards of the block are present, and with +/// [`MediaFecError::Invalid`] if the records are not from one block. +pub fn recover_video_block( + parity: &[GameCultMediaVideoParityShardRecord], + data: &[GameCultMediaVideoAccessUnitRecord], +) -> Result, MediaFecError> { + let geometry = parity + .first() + .ok_or_else(|| MediaFecError::invalid("recovery needs at least one parity shard"))?; + let k = usize::from(geometry.block_data_count); + let m = usize::from(geometry.parity_count); + let start = usize::from(geometry.block_data_start); + let shard_bytes = geometry.shard_payload_bytes as usize; + + let mut shards: Vec>> = vec![None; k + m]; + for shard in parity { + validate_video_parity_record(shard).map_err(from_anyhow)?; + if !same_block(geometry, shard) { + return Err(MediaFecError::invalid("parity shards are not from one block")); + } + shards[k + usize::from(shard.parity_index)].get_or_insert_with(|| shard.payload.clone()); + } + for chunk in data { + validate_video_record(chunk).map_err(from_anyhow)?; + let index = usize::from(chunk.chunk_index); + let framing_agrees = chunk.stream_id == geometry.stream_id + && chunk.session_id == geometry.session_id + && chunk.frame_id == geometry.frame_id + && chunk.chunk_count == geometry.chunk_count; + if !framing_agrees || index < start || index >= start + k { + return Err(MediaFecError::invalid("data chunk does not belong to the parity's block")); + } + let expected = if index == start + k - 1 { + geometry.last_chunk_payload_bytes as usize + } else { + shard_bytes + }; + if chunk.payload.len() != expected { + return Err(MediaFecError::invalid(format!( + "chunk {index} is {} bytes but the block declares {expected}", + chunk.payload.len() + ))); + } + shards[index - start].get_or_insert_with(|| padded(&chunk.payload, shard_bytes)); + } + + let missing: Vec = (0..k).filter(|&slot| shards[slot].is_none()).collect(); + let recovered = recover_data_shards(k, shards)?; + Ok(missing + .into_iter() + .map(|slot| { + let mut payload = recovered[slot].clone(); + if slot == k - 1 { + payload.truncate(geometry.last_chunk_payload_bytes as usize); + } + GameCultMediaVideoAccessUnitRecord { + stream_id: geometry.stream_id.clone(), + session_id: geometry.session_id.clone(), + frame_id: geometry.frame_id, + codec: geometry.codec.clone(), + pts_ticks: geometry.pts_ticks, + duration_ticks: geometry.duration_ticks, + timebase_num: geometry.timebase_num, + timebase_den: geometry.timebase_den, + keyframe: geometry.keyframe, + dependency_frame_id: geometry.dependency_frame_id, + deadline_ticks: geometry.deadline_ticks, + chunk_index: (start + slot) as u16, + chunk_count: geometry.chunk_count, + payload, + } + }) + .collect()) +} + +fn same_block(a: &GameCultMediaVideoParityShardRecord, b: &GameCultMediaVideoParityShardRecord) -> bool { + a.stream_id == b.stream_id + && a.session_id == b.session_id + && a.frame_id == b.frame_id + && a.codec == b.codec + && a.pts_ticks == b.pts_ticks + && a.duration_ticks == b.duration_ticks + && a.timebase_num == b.timebase_num + && a.timebase_den == b.timebase_den + && a.keyframe == b.keyframe + && a.dependency_frame_id == b.dependency_frame_id + && a.deadline_ticks == b.deadline_ticks + && a.chunk_count == b.chunk_count + && a.fec_scheme == b.fec_scheme + && a.block_index == b.block_index + && a.block_count == b.block_count + && a.block_data_start == b.block_data_start + && a.block_data_count == b.block_data_count + && a.parity_count == b.parity_count + && a.shard_payload_bytes == b.shard_payload_bytes + && a.last_chunk_payload_bytes == b.last_chunk_payload_bytes +} + +// --------------------------------------------------------------------------- +// Audio +// --------------------------------------------------------------------------- + +/// Protects one block of audio packets: exactly `policy.audio_data_shards` +/// packets with contiguous ids and presentation times, one payload length, and +/// one codec and timebase. Returns the parity records in `parity_index` order. +/// Errors if a parity record, wrapped with `provenance`, exceeds +/// `policy.max_wire_bytes`; the packets are smaller than their parity. +pub fn protect_audio_block( + packets: &[GameCultMediaAudioPacketRecord], + policy: &MediaFecPolicy, + provenance: MediaWireProvenance<'_>, +) -> Result, MediaFecError> { + policy.validate()?; + if packets.len() != usize::from(policy.audio_data_shards) { + return Err(MediaFecError::invalid(format!( + "an audio block is {} packets, got {}", + policy.audio_data_shards, + packets.len() + ))); + } + let first = &packets[0]; + let shard_bytes = first.payload.len(); + for (index, packet) in packets.iter().enumerate() { + validate_audio_record(packet).map_err(from_anyhow)?; + let expected = + audio_slot(first.packet_id, first.pts_ticks, first.duration_ticks, index as u64); + if packet.stream_id != first.stream_id + || packet.session_id != first.session_id + || packet.codec != first.codec + || packet.timebase_num != first.timebase_num + || packet.timebase_den != first.timebase_den + || packet.duration_ticks != first.duration_ticks + || Some((packet.packet_id, packet.pts_ticks)) != expected + || packet.payload.len() != shard_bytes + { + return Err(MediaFecError::invalid( + "audio block packets must be contiguous and share codec, timebase, duration and payload length", + )); + } + } + let data: Vec> = packets.iter().map(|packet| packet.payload.clone()).collect(); + let deadline_ticks = packets.iter().map(|packet| packet.deadline_ticks).max().unwrap_or(0); + let parity: Vec = + parity_shards(&data, usize::from(policy.audio_parity_shards))? + .into_iter() + .enumerate() + .map(|(parity_index, payload)| GameCultMediaAudioParityShardRecord { + stream_id: first.stream_id.clone(), + session_id: first.session_id.clone(), + codec: first.codec.clone(), + fec_scheme: MEDIA_FEC_SCHEME_RS_GF256_V1.to_string(), + base_packet_id: first.packet_id, + base_pts_ticks: first.pts_ticks, + packet_duration_ticks: first.duration_ticks, + timebase_num: first.timebase_num, + timebase_den: first.timebase_den, + deadline_ticks, + data_shard_count: policy.audio_data_shards, + parity_index: parity_index as u16, + parity_shard_count: policy.audio_parity_shards, + shard_payload_bytes: shard_bytes as u32, + payload, + }) + .collect(); + // A parity record repeats every field of a packet and adds the scheme and + // the geometry, so it is the larger of the block's records: if it fits, the + // packets do. + for shard in &parity { + fits_datagram(&GameCultMediaWireRecord::AudioParity(shard.clone()), provenance, policy)?; + } + Ok(parity) +} + +/// Rebuilds the packets an audio block lost, ordered by `packet_id`. A +/// recovered packet takes its id and presentation time from the block's +/// arithmetic and its deadline from the block's, the latest in the block: +/// gate playout on `pts_ticks`, not `deadline_ticks`. Errors as [`recover_video_block`] does. +pub fn recover_audio_block( + parity: &[GameCultMediaAudioParityShardRecord], + data: &[GameCultMediaAudioPacketRecord], +) -> Result, MediaFecError> { + let geometry = parity + .first() + .ok_or_else(|| MediaFecError::invalid("recovery needs at least one parity shard"))?; + let k = usize::from(geometry.data_shard_count); + let m = usize::from(geometry.parity_shard_count); + let shard_bytes = geometry.shard_payload_bytes as usize; + + let mut shards: Vec>> = vec![None; k + m]; + for shard in parity { + validate_audio_parity_record(shard).map_err(from_anyhow)?; + if !same_audio_block(geometry, shard) { + return Err(MediaFecError::invalid("parity shards are not from one block")); + } + shards[k + usize::from(shard.parity_index)].get_or_insert_with(|| shard.payload.clone()); + } + for packet in data { + validate_audio_record(packet).map_err(from_anyhow)?; + let slot = packet + .packet_id + .checked_sub(geometry.base_packet_id) + .filter(|slot| *slot < k as u64) + .ok_or_else(|| MediaFecError::invalid("audio packet does not belong to the parity's block"))? + as usize; + if packet.stream_id != geometry.stream_id + || packet.session_id != geometry.session_id + || packet.payload.len() != shard_bytes + { + return Err(MediaFecError::invalid("audio packet disagrees with the parity's block")); + } + shards[slot].get_or_insert_with(|| packet.payload.clone()); + } + + let missing: Vec = (0..k).filter(|&slot| shards[slot].is_none()).collect(); + let recovered = recover_data_shards(k, shards)?; + missing + .into_iter() + .map(|slot| { + let (packet_id, pts_ticks) = audio_slot( + geometry.base_packet_id, + geometry.base_pts_ticks, + geometry.packet_duration_ticks, + slot as u64, + ) + .ok_or_else(|| MediaFecError::invalid("audio block runs past the packet id or pts range"))?; + Ok(GameCultMediaAudioPacketRecord { + stream_id: geometry.stream_id.clone(), + session_id: geometry.session_id.clone(), + packet_id, + codec: geometry.codec.clone(), + pts_ticks, + duration_ticks: geometry.packet_duration_ticks, + timebase_num: geometry.timebase_num, + timebase_den: geometry.timebase_den, + deadline_ticks: geometry.deadline_ticks, + payload: recovered[slot].clone(), + }) + }) + .collect() +} + +fn same_audio_block(a: &GameCultMediaAudioParityShardRecord, b: &GameCultMediaAudioParityShardRecord) -> bool { + a.stream_id == b.stream_id + && a.session_id == b.session_id + && a.codec == b.codec + && a.fec_scheme == b.fec_scheme + && a.base_packet_id == b.base_packet_id + && a.base_pts_ticks == b.base_pts_ticks + && a.packet_duration_ticks == b.packet_duration_ticks + && a.timebase_num == b.timebase_num + && a.timebase_den == b.timebase_den + && a.deadline_ticks == b.deadline_ticks + && a.data_shard_count == b.data_shard_count + && a.parity_shard_count == b.parity_shard_count + && a.shard_payload_bytes == b.shard_payload_bytes +} diff --git a/packages/cultnet-rs/src/media_stream_contracts.rs b/packages/cultnet-rs/src/media_stream_contracts.rs index 5704912e..2991bb43 100644 --- a/packages/cultnet-rs/src/media_stream_contracts.rs +++ b/packages/cultnet-rs/src/media_stream_contracts.rs @@ -36,15 +36,15 @@ //! Nothing here specifies a delivery guarantee. Whether a stream rides a //! reliable or lossy CultNet channel, and whether it carries parity, is a //! transport decision made per stream — not a property of the payload shape. -//! Video parity shards exist for producers that choose forward error correction; -//! audio currently has no parity record, and a producer emitting audio on a -//! lossy channel needs one. +//! Video and audio parity shards exist for producers that choose forward error +//! correction; [`crate::media_fec`] owns the code that makes and consumes them. use cultcache_rs::DatabaseEntry; pub const GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA: &str = "gamecult.media_video_access_unit.v1"; -pub const GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "gamecult.media_video_parity_shard.v2"; +pub const GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "gamecult.media_video_parity_shard.v3"; pub const GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA: &str = "gamecult.media_audio_packet.v1"; +pub const GAMECULT_MEDIA_AUDIO_PARITY_SHARD_SCHEMA: &str = "gamecult.media_audio_parity_shard.v1"; pub const GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA: &str = "gamecult.media_receiver_feedback.v1"; pub const GAMECULT_MEDIA_STREAM_ADVERTISEMENT_SCHEMA: &str = "gamecult.media_stream_advertisement.v1"; @@ -464,15 +464,25 @@ pub struct GameCultMediaVideoAccessUnitRecord { pub payload: Vec, } -/// Forward-error-correction parity for one video access unit. Carries the -/// framing fields of the access unit it protects so a receiver can recover -/// without having seen the original, plus the shard geometry needed to run -/// recovery: `parity_index` within `parity_count`, and the chunk sizes the -/// parity was computed over. +/// Forward-error-correction parity for one block of a video access unit. +/// +/// A frame's `chunk_count` chunks are split into `block_count` contiguous +/// blocks, and each block carries `parity_count` parity shards computed over its +/// `block_data_count` data chunks (`block_data_start..block_data_start + +/// block_data_count`). Any `block_data_count` of a block's +/// `block_data_count + parity_count` shards recover it. The math is +/// [`crate::media_fec`]'s and is named by `fec_scheme`; a decoder that does not +/// know the scheme must drop the shard rather than guess. +/// +/// It carries the framing fields of the access unit it protects, so a receiver +/// can rebuild a lost chunk without having seen any chunk of the frame. +/// `shard_payload_bytes` is the padded shard length; `last_chunk_payload_bytes` +/// is the true length of the block's final data chunk, the only one that may be +/// shorter. #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "gamecult.media_video_parity_shard", - schema = "gamecult.media_video_parity_shard.v2" + schema = "gamecult.media_video_parity_shard.v3" )] pub struct GameCultMediaVideoParityShardRecord { #[cultcache(key = 0)] @@ -500,14 +510,26 @@ pub struct GameCultMediaVideoParityShardRecord { #[cultcache(key = 11)] pub chunk_count: u16, #[cultcache(key = 12)] - pub parity_index: u16, + pub fec_scheme: String, #[cultcache(key = 13)] - pub parity_count: u16, + pub block_index: u16, #[cultcache(key = 14)] - pub chunk_payload_bytes: u32, + pub block_count: u16, #[cultcache(key = 15)] + pub block_data_start: u16, + /// `k`: how many data chunks this block protects. + #[cultcache(key = 16)] + pub block_data_count: u16, + #[cultcache(key = 17)] + pub parity_index: u16, + /// `m`: how many parity shards this block carries. + #[cultcache(key = 18)] + pub parity_count: u16, + #[cultcache(key = 19)] + pub shard_payload_bytes: u32, + #[cultcache(key = 20)] pub last_chunk_payload_bytes: u32, - #[cultcache(key = 16, bytes)] + #[cultcache(key = 21, bytes)] pub payload: Vec, } @@ -541,6 +563,60 @@ pub struct GameCultMediaAudioPacketRecord { pub payload: Vec, } +/// Forward-error-correction parity for a block of consecutive audio packets. +/// +/// The block is `data_shard_count` packets with contiguous ids starting at +/// `base_packet_id`, all with one payload length (`shard_payload_bytes`) and +/// contiguous presentation times spaced `packet_duration_ticks` apart. A packet +/// recovered from parity takes its id and time from that arithmetic, and its +/// `deadline_ticks` from the block's, which is the latest in the block: the +/// per-packet deadline is not recoverable, so a recovered packet may be held +/// past its own but never past the block's. +/// +/// Codecs whose frames vary in length pad each one to a shared shard length +/// before parity is computed; that framing belongs to the codec, not to this +/// record. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_audio_parity_shard", + schema = "gamecult.media_audio_parity_shard.v1" +)] +pub struct GameCultMediaAudioParityShardRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub session_id: String, + #[cultcache(key = 2)] + pub codec: String, + #[cultcache(key = 3)] + pub fec_scheme: String, + #[cultcache(key = 4)] + pub base_packet_id: u64, + #[cultcache(key = 5)] + pub base_pts_ticks: i64, + #[cultcache(key = 6)] + pub packet_duration_ticks: u32, + #[cultcache(key = 7)] + pub timebase_num: u32, + #[cultcache(key = 8)] + pub timebase_den: u32, + /// The latest `deadline_ticks` among the block's data packets. + #[cultcache(key = 9)] + pub deadline_ticks: i64, + /// `k`. + #[cultcache(key = 10)] + pub data_shard_count: u16, + #[cultcache(key = 11)] + pub parity_index: u16, + /// `m`. + #[cultcache(key = 12)] + pub parity_shard_count: u16, + #[cultcache(key = 13)] + pub shard_payload_bytes: u32, + #[cultcache(key = 14, bytes)] + pub payload: Vec, +} + /// What a receiver observed, published back to the producer so it can adapt: /// what decoded, what is missing, what arrived too late, and how much queue and /// jitter the receiver is carrying. diff --git a/packages/cultnet-rs/src/media_stream_wire.rs b/packages/cultnet-rs/src/media_stream_wire.rs index 9b7fbc1b..5c89a8c3 100644 --- a/packages/cultnet-rs/src/media_stream_wire.rs +++ b/packages/cultnet-rs/src/media_stream_wire.rs @@ -28,20 +28,32 @@ use crate::contracts::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetWireContract, decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, }; +use crate::media_fec::{MEDIA_FEC_SCHEME_RS_GF256_V1, audio_slot}; use crate::media_stream_contracts::{ - GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, - GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, - GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_AUDIO_PARITY_SHARD_SCHEMA, + GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, + GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, GameCultMediaAudioPacketRecord, + GameCultMediaAudioParityShardRecord, GameCultMediaReceiverFeedbackRecord, GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, }; -/// The CultNet channel video and receiver feedback ride. +/// The CultNet channel every media record rides: video, audio, their parity, +/// and receiver feedback. Whether it is reliable is the transport profile's +/// choice per session; stale media is dropped, not retransmitted. pub const GAMECULT_MEDIA_CHANNEL: &str = "media"; -/// The CultNet channel audio rides. Audio has its own channel so one session -/// can carry it reliably with a short expiry while video stays lossy under -/// parity; the RUDP transport profile owns each channel's delivery. -pub const GAMECULT_MEDIA_AUDIO_CHANNEL: &str = "audio"; +/// The largest wrapped media record the decoder accepts, and the most the RUDP +/// `media` channel reassembles: 64 KiB. +/// +/// It is the bound a receiver holds a peer to. A parity shard's length is +/// whatever its producer's [`MediaFecPolicy::max_wire_bytes`](crate::MediaFecPolicy::max_wire_bytes) allowed, and +/// recovery work grows with the square of the block size times that length, so +/// without a ceiling one hostile record could name any length RUDP would +/// reassemble (16 MiB). 64 KiB is one maximum UDP datagram: it covers every +/// path MTU in use, jumbo frames (9,000 bytes) with a wide margin, and a +/// record is still one message rather than a stream. A policy may not ask for +/// more; see [`MediaFecPolicy`](crate::MediaFecPolicy). +pub const GAMECULT_MEDIA_MAX_WIRE_BYTES: usize = 64 * 1024; /// One media record on its way somewhere. #[derive(Clone, Debug, PartialEq, Eq)] @@ -49,6 +61,7 @@ pub enum GameCultMediaWireRecord { Video(GameCultMediaVideoAccessUnitRecord), VideoParity(GameCultMediaVideoParityShardRecord), Audio(GameCultMediaAudioPacketRecord), + AudioParity(GameCultMediaAudioParityShardRecord), Feedback(GameCultMediaReceiverFeedbackRecord), } @@ -84,6 +97,7 @@ impl GameCultMediaWireRecord { Self::Video(_) => GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, Self::VideoParity(_) => GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, Self::Audio(_) => GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, + Self::AudioParity(_) => GAMECULT_MEDIA_AUDIO_PARITY_SHARD_SCHEMA, Self::Feedback(_) => GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, } } @@ -96,13 +110,21 @@ impl GameCultMediaWireRecord { record.stream_id, record.session_id, record.frame_id, record.chunk_index ), Self::VideoParity(record) => format!( - "{}:{}:video-parity:{}:{}", - record.stream_id, record.session_id, record.frame_id, record.parity_index + "{}:{}:video-parity:{}:{}:{}", + record.stream_id, + record.session_id, + record.frame_id, + record.block_index, + record.parity_index ), Self::Audio(record) => format!( "{}:{}:audio:{}", record.stream_id, record.session_id, record.packet_id ), + Self::AudioParity(record) => format!( + "{}:{}:audio-parity:{}:{}", + record.stream_id, record.session_id, record.base_packet_id, record.parity_index + ), Self::Feedback(record) => format!( "{}:{}:feedback:{}", record.stream_id, record.session_id, record.receiver_id @@ -124,6 +146,7 @@ impl GameCultMediaWireRecord { Self::Video(record) => rmp_serde::to_vec(record), Self::VideoParity(record) => rmp_serde::to_vec(record), Self::Audio(record) => rmp_serde::to_vec(record), + Self::AudioParity(record) => rmp_serde::to_vec(record), Self::Feedback(record) => rmp_serde::to_vec(record), } .map_err(Into::into) @@ -159,11 +182,24 @@ pub fn encode_media_wire_record( }, }; - encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).map_err(Into::into) + let wire = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?; + if wire.len() > GAMECULT_MEDIA_MAX_WIRE_BYTES { + return Err(anyhow!( + "media record is {} bytes, over the {GAMECULT_MEDIA_MAX_WIRE_BYTES}-byte ceiling", + wire.len() + )); + } + Ok(wire) } /// Unwraps a media record from the wire. pub fn decode_media_wire_record(payload: &[u8]) -> Result { + if payload.len() > GAMECULT_MEDIA_MAX_WIRE_BYTES { + return Err(anyhow!( + "media record is {} bytes, over the {GAMECULT_MEDIA_MAX_WIRE_BYTES}-byte ceiling", + payload.len() + )); + } let message = decode_cultnet_message_from_slice(payload, CultNetWireContract::CultNetSchemaV0)?; let CultNetMessage::DocumentPutRaw { document, .. } = message else { return Err(anyhow!("expected cultnet.document_put_raw.v0")); @@ -191,6 +227,12 @@ pub fn decode_media_wire_record(payload: &[u8]) -> Result { + let record: GameCultMediaAudioParityShardRecord = + rmp_serde::from_slice(&document.payload)?; + validate_audio_parity_record(&record)?; + GameCultMediaWireRecord::AudioParity(record) + } GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA => { let record: GameCultMediaReceiverFeedbackRecord = rmp_serde::from_slice(&document.payload)?; @@ -349,39 +391,114 @@ pub fn validate_video_parity_record(record: &GameCultMediaVideoParityShardRecord "video parity media record deadline_ticks must not precede pts_ticks" )); } + if record.fec_scheme != MEDIA_FEC_SCHEME_RS_GF256_V1 { + return Err(anyhow!( + "video parity media record fec_scheme {:?} is not supported", + record.fec_scheme + )); + } if record.chunk_count == 0 { return Err(anyhow!( "video parity media record chunk_count must be non-zero" )); } + if record.block_count == 0 || record.block_index >= record.block_count { + return Err(anyhow!( + "video parity media record block_index {} is outside block_count {}", + record.block_index, + record.block_count + )); + } + if record.block_data_count == 0 + || u32::from(record.block_data_start) + u32::from(record.block_data_count) + > u32::from(record.chunk_count) + { + return Err(anyhow!( + "video parity media record block does not lie inside the frame's chunks" + )); + } if record.parity_count == 0 || record.parity_index >= record.parity_count { return Err(anyhow!( - "video parity media record has invalid parity stripe metadata" + "video parity media record parity_index {} is outside parity_count {}", + record.parity_index, + record.parity_count )); } - if record.parity_count > record.chunk_count { + if usize::from(record.block_data_count) + usize::from(record.parity_count) > 256 { return Err(anyhow!( - "video parity media record parity_count exceeds chunk_count" + "video parity media record block exceeds 256 shards" )); } - if record.chunk_payload_bytes == 0 || record.last_chunk_payload_bytes == 0 { + if record.shard_payload_bytes == 0 { return Err(anyhow!( - "video parity media record chunk lengths must be non-zero" + "video parity media record shard_payload_bytes must be non-zero" )); } - if record.last_chunk_payload_bytes > record.chunk_payload_bytes { + if record.last_chunk_payload_bytes == 0 + || record.last_chunk_payload_bytes > record.shard_payload_bytes + { return Err(anyhow!( - "video parity media record last chunk length exceeds regular chunk length" + "video parity media record last chunk length must be in 1..=shard_payload_bytes" )); } - if record.payload.is_empty() { + if record.payload.len() != record.shard_payload_bytes as usize { return Err(anyhow!( - "video parity media record payload must be non-empty" + "video parity media record payload does not match declared shard length" )); } - if record.payload.len() > record.chunk_payload_bytes as usize { + Ok(()) +} + +pub fn validate_audio_parity_record(record: &GameCultMediaAudioParityShardRecord) -> Result<()> { + if record.stream_id.is_empty() || record.session_id.is_empty() || record.codec.is_empty() { return Err(anyhow!( - "video parity media record payload exceeds declared chunk length" + "audio parity media record stream_id, session_id and codec must be non-empty" + )); + } + if record.timebase_num == 0 || record.timebase_den == 0 || record.packet_duration_ticks == 0 { + return Err(anyhow!( + "audio parity media record timing metadata must be non-zero" + )); + } + if record.fec_scheme != MEDIA_FEC_SCHEME_RS_GF256_V1 { + return Err(anyhow!( + "audio parity media record fec_scheme {:?} is not supported", + record.fec_scheme + )); + } + if record.data_shard_count == 0 + || record.parity_shard_count == 0 + || record.parity_index >= record.parity_shard_count + || usize::from(record.data_shard_count) + usize::from(record.parity_shard_count) > 256 + { + return Err(anyhow!("audio parity media record stripe metadata is invalid")); + } + let Some((_, last_pts_ticks)) = audio_slot( + record.base_packet_id, + record.base_pts_ticks, + record.packet_duration_ticks, + u64::from(record.data_shard_count) - 1, + ) else { + return Err(anyhow!( + "audio parity media record block runs past the packet id or pts range" + )); + }; + // Recovery gives every recovered packet the block's deadline, so it must + // not precede the last packet's pts or a recovered packet would fail + // `validate_audio_record`. + if record.deadline_ticks < last_pts_ticks { + return Err(anyhow!( + "audio parity media record deadline_ticks must not precede the block's last pts_ticks" + )); + } + if record.shard_payload_bytes == 0 { + return Err(anyhow!( + "audio parity media record shard_payload_bytes must be non-zero" + )); + } + if record.payload.len() != record.shard_payload_bytes as usize { + return Err(anyhow!( + "audio parity media record payload does not match declared shard length" )); } Ok(()) diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 3b348fe3..3ee6af66 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -8,7 +8,6 @@ use std::net::SocketAddr; use std::net::UdpSocket; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use uuid::Uuid; use crate::CultNetMessage; use crate::CultNetReconnectController; @@ -23,13 +22,15 @@ use crate::CultNetTransportProfile; use crate::CultNetTransportProtocol; use crate::CultNetTransportStats; use crate::CultNetWireContract; +use crate::GAMECULT_MEDIA_CHANNEL; +use crate::GAMECULT_MEDIA_MAX_WIRE_BYTES; use crate::create_reconnect_policy; use crate::decode_cultnet_message_from_slice; use crate::encode_cultnet_message_to_vec; const RUDP_MAGIC: [u8; 4] = [0x43, 0x4e, 0x52, 0x30]; const RUDP_VERSION: u8 = 0; -const RUDP_FIXED_HEADER_BYTES: usize = 36; +pub(crate) const RUDP_FIXED_HEADER_BYTES: usize = 36; pub const CULTNET_RUDP_DEFAULT_MAX_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; pub const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS: usize = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW: usize = 4_096; @@ -44,6 +45,12 @@ const RUDP_MAX_TRACKED_CHANNELS: usize = 64; /// ahead of the current highest would otherwise advance the receive state past /// every legitimate sequence still in flight and starve the session. const RUDP_RECEIVE_AHEAD_WINDOW: u32 = 1_024; +/// How long a client keeps retransmitting a Connect nobody answered before it +/// abandons that attempt for a fresh one. A Connect the server answers with an +/// Ack, never an Accept, is one the server judged stale (see +/// `accept_connect`); retransmitting the same sequence would never change its +/// mind. +const RUDP_CONNECT_ATTEMPT_MS: u64 = 3_000; const RUDP_MAX_ORDERED_BUFFERED_FRAMES: usize = 1_024; const RUDP_MAX_ORDERED_BUFFERED_BYTES: usize = 4 * 1024 * 1024; @@ -96,7 +103,10 @@ pub struct CultNetRudpReceiveResult { #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpSessionOptions { pub connection_id: u32, - pub initial_sequence: u32, + /// The first sequence this session issues. Unset, the session draws its + /// own at random: every session is a new identity to its peer, and an + /// options object reused for several must not give them one sequence. + pub initial_sequence: Option, pub resend_delay_ms: u64, pub max_pending_reliable_packets: Option, } @@ -105,13 +115,41 @@ impl Default for CultNetRudpSessionOptions { fn default() -> Self { Self { connection_id: 0, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, max_pending_reliable_packets: None, } } } +/// A sequence drawn at random from [1, 2^31). The Connect's sequence is what +/// tells a peer whether a Connect repeats one it already accepted or starts a +/// new session, so two sessions must not share one by default. At least 2^31 +/// sequences remain before the space is exhausted. +fn draw_sequence() -> u32 { + rand::random_range(1..(1_u32 << 31)) +} + +/// Whether `sequence` is at or before `mark` in serial order, within the +/// receive window. The compare is modular, so it holds across the wrap of the +/// 32-bit space; a sequence further back than the window is the duplicate +/// test's below-window clause, and one ahead of the mark is never before it. +fn at_or_before(sequence: u32, mark: u32) -> bool { + mark.wrapping_sub(sequence) < RUDP_RECEIVED_SEQUENCE_WINDOW as u32 +} + +/// Whether the packet's ack field, or its mask, names `sequence`. +fn packet_acknowledges(packet: &CultNetRudpPacket, sequence: u32) -> bool { + if packet.ack == sequence { + return true; + } + (0..32).any(|bit| { + packet.ack_mask & (1_u32 << bit) != 0 + && packet.ack > bit + && packet.ack - bit - 1 == sequence + }) +} + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct CultNetRudpSendOptions { pub reliable: bool, @@ -126,13 +164,13 @@ pub struct CultNetRudpSendOptions { } /// Identifies every transport packet belonging to one non-expiring reliable -/// send. Receipts are local to the session that issued them: a peer reset or -/// a Disconnect received from the peer invalidates them for good. A local -/// `disconnect()` or a timeout does not; the writes it leaves owed carry into -/// the next session. +/// send. A write belongs to the session generation it was issued in, and every +/// way a generation can end (a Disconnect from the peer, a local `disconnect()`, +/// a timeout, a refused packet) invalidates the receipts it issued for good. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpReliableSendReceipt { - session_scope: Uuid, + session_id: u64, + generation: u64, sequences: Vec, } @@ -156,12 +194,6 @@ struct PendingReliablePacket { expires_at_ms: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] -struct PendingOrderedFrame { - frame: CultNetRudpDeliveredFrame, - next_sequence: u32, -} - #[derive(Clone, Debug, PartialEq, Eq)] struct FragmentBuffer { channel_id: String, @@ -176,7 +208,28 @@ struct FragmentBuffer { } pub struct CultNetRudpSession { - session_scope: Uuid, + /// Random per session. A receipt carries its session's id, so one asked of + /// another session reads `Invalidated` instead of borrowing that session's + /// generation numbers. + session_id: u64, + /// Advances every time a generation ends. Everything issued in a generation + /// (writes, receipts, flushes) belongs to it and dies with it. + generation: u64, + /// True from the end of a generation until the next Connect or Accept begins + /// one. A flush started in that interval has no live generation to wait on. + ended: bool, + /// The sequence of the Connect that started the current generation: sent + /// by this side, or accepted from the peer. A Connect repeats exactly when + /// the session is connected and the Connect carries this sequence. + connect_sequence: Option, + /// A Connect this side sent is unanswered. Only then is an Accept honoured. + awaiting_accept: bool, + /// The payload of the Connect this side sent, kept so a fresh attempt can + /// always be built whatever became of the pending packet. + connect_payload: Vec, + /// When the unanswered Connect first went out; `due_resends` abandons it + /// for a fresh attempt once `RUDP_CONNECT_ATTEMPT_MS` has passed. + connect_started_at_ms: u64, connection_id: u32, resend_delay_ms: u64, max_pending_reliable_packets: Option, @@ -193,8 +246,15 @@ pub struct CultNetRudpSession { pending_reliable: BTreeMap, queued_reliable: VecDeque<(CultNetRudpPacket, Option)>, reliable_packets_expired: u64, - ordered_next_sequence_by_channel: BTreeMap, - ordered_buffers: BTreeMap>, + /// Every reliable sequence up to and including this one has been received + /// since the peer state was last reset. Only the handshake seeds it: the + /// peer's Connect on the accepting side, the peer's Accept on the + /// connecting side. Reliable data that arrives before that is refused. + received_through: Option, + /// Ordered frames received but not yet deliverable, keyed by first + /// sequence. A frame is held for exactly one reason: a reliable sequence + /// below it has not arrived. + ordered_held: BTreeMap, fragment_buffers: BTreeMap<(String, u16), FragmentBuffer>, fragment_touches: u64, fragment_sets_evicted: u64, @@ -203,13 +263,19 @@ pub struct CultNetRudpSession { impl CultNetRudpSession { pub fn new(options: CultNetRudpSessionOptions) -> Self { Self { - session_scope: Uuid::new_v4(), + session_id: rand::random(), + generation: 0, + ended: false, + connect_sequence: None, + awaiting_accept: false, + connect_payload: Vec::new(), + connect_started_at_ms: 0, connection_id: options.connection_id, resend_delay_ms: options.resend_delay_ms, max_pending_reliable_packets: options.max_pending_reliable_packets, max_payload_bytes: None, max_pending_fragment_sets: 64, - next_sequence: options.initial_sequence, + next_sequence: options.initial_sequence.unwrap_or_else(draw_sequence), next_sequenced_by_channel: BTreeMap::new(), next_fragment_id: 1, connected: false, @@ -220,8 +286,8 @@ impl CultNetRudpSession { pending_reliable: BTreeMap::new(), queued_reliable: VecDeque::new(), reliable_packets_expired: 0, - ordered_next_sequence_by_channel: BTreeMap::new(), - ordered_buffers: BTreeMap::new(), + received_through: None, + ordered_held: BTreeMap::new(), fragment_buffers: BTreeMap::new(), fragment_touches: 0, fragment_sets_evicted: 0, @@ -242,6 +308,7 @@ impl CultNetRudpSession { pub fn assume_connected(&mut self, now_ms: u64) { self.connected = true; + self.ended = false; self.last_received_at_ms = Some(now_ms); } @@ -280,15 +347,35 @@ impl CultNetRudpSession { Some(pending.packet.clone()) } - fn session_scope(&self) -> Uuid { - self.session_scope + fn generation(&self) -> u64 { + self.generation + } + + fn ended(&self) -> bool { + self.ended + } + + /// The one way a session generation ends. The session stops being + /// connected and what it still owed the peer dies with it: a write not yet + /// acknowledged is dropped, so no later session retransmits it or credits an + /// ack to it, and its receipt can never become Acknowledged. What was + /// learned from the peer is not touched: the peer may not know the session + /// ended, and forgetting what it sent would let its retransmits be + /// delivered twice. + fn end_session(&mut self) { + self.connected = false; + self.ended = true; + self.awaiting_accept = false; + self.generation += 1; + self.pending_reliable.clear(); + self.queued_reliable.clear(); } fn reliable_send_status( &self, receipt: &CultNetRudpReliableSendReceipt, ) -> CultNetRudpReliableSendStatus { - if receipt.session_scope != self.session_scope { + if receipt.session_id != self.session_id || receipt.generation != self.generation { return CultNetRudpReliableSendStatus::Invalidated; } if receipt @@ -312,7 +399,8 @@ impl CultNetRudpSession { )); } Ok(CultNetRudpReliableSendReceipt { - session_scope: self.session_scope, + session_id: self.session_id, + generation: self.generation, sequences: packets.iter().map(|packet| packet.sequence).collect(), }) } @@ -342,17 +430,14 @@ impl CultNetRudpSession { /// ids already issued stay issued: a peer that still remembers them must /// never see one reused. pub fn reset_peer_state(&mut self) { - self.session_scope = Uuid::new_v4(); + self.end_session(); self.next_sequenced_by_channel.clear(); - self.connected = false; self.last_received_at_ms = None; self.highest_received_sequence = None; self.received_sequences.clear(); self.latest_sequenced_by_channel.clear(); - self.pending_reliable.clear(); - self.queued_reliable.clear(); - self.ordered_next_sequence_by_channel.clear(); - self.ordered_buffers.clear(); + self.received_through = None; + self.ordered_held.clear(); self.fragment_buffers.clear(); } @@ -367,6 +452,12 @@ impl CultNetRudpSession { } pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { + // A session that has had a peer starts a new generation: nothing it + // learned from that peer describes the one this Connect reaches, and + // what it still owed that peer no longer takes room in the queue. + if self.connect_sequence.is_some() { + self.reset_peer_state(); + } self.ensure_reliable_capacity(1)?; let packet = self.create_packet( CultNetRudpPacketType::Connect, @@ -376,10 +467,49 @@ impl CultNetRudpSession { true, false, ); + self.ended = false; + self.connect_sequence = Some(packet.sequence); + self.awaiting_accept = true; + self.connect_started_at_ms = now_ms; + self.connect_payload = packet.payload.clone(); self.track_reliable(packet.clone(), now_ms, None); Ok(packet) } + /// Whether `packet` is a Connect this session's current generation already + /// owns: a retransmit of the Connect that started it (the session is + /// connected and the sequence is that Connect's), or a stale copy of an + /// earlier attempt by the same client (a sequence before it, within the + /// receive window). Servers that keep one session per peer ask this to tell + /// a Connect that starts a new session from one that does not; anything + /// else that reaches `accept_connect` starts a new generation. + pub fn connect_repeats(&self, packet: &CultNetRudpPacket) -> bool { + packet.packet_type == CultNetRudpPacketType::Connect + && self.connected + && self.connect_sequence.is_some_and(|connect| { + connect == packet.sequence || self.connect_is_stale(packet.sequence, connect) + }) + } + + /// A Connect that precedes the current generation's within the receive + /// window is the client's earlier attempt, delayed in the network: a + /// client that retried never sends a lower sequence again. Restarting on + /// it would strand the client, which honours only the Accept for its + /// newest Connect. The rule is TCP's answer to a delayed SYN (RFC 5961's + /// challenge ACK): keep the connection and answer with an Ack. A restarted + /// client whose random initial sequence lands in this window is answered + /// the same way and abandons the attempt for a fresh draw + /// (`RUDP_CONNECT_ATTEMPT_MS`). + fn connect_is_stale(&self, sequence: u32, current: u32) -> bool { + sequence != current && at_or_before(sequence, current) + } + + /// Answers a Connect. A repeat of the accepted one queues nothing, so a + /// Connect storm cannot grow the reliable queue: the reply is the Accept + /// still awaiting acknowledgement, or an Ack once it was acknowledged. A + /// stale copy of an earlier attempt gets the same reply and changes + /// nothing else: it is not evidence the peer is alive. Any other Connect + /// ends the current generation, forgets the peer and accepts a new one. pub fn accept_connect( &mut self, packet: &CultNetRudpPacket, @@ -394,10 +524,24 @@ impl CultNetRudpSession { )); } + if self.connect_repeats(packet) { + if self.connect_sequence == Some(packet.sequence) { + self.apply_acknowledgements(packet); + self.remember_received(packet.sequence); + self.last_received_at_ms = Some(now_ms); + } + return Ok(self + .pending_accept_for_resend(now_ms) + .unwrap_or_else(|| self.create_ack())); + } + + self.reset_peer_state(); self.ensure_reliable_capacity(1)?; - self.remember_received(packet.sequence); + self.seed_received(packet.sequence); self.last_received_at_ms = Some(now_ms); + self.connect_sequence = Some(packet.sequence); self.connected = true; + self.ended = false; let response = self.create_packet( CultNetRudpPacketType::Accept, "control", @@ -410,30 +554,6 @@ impl CultNetRudpSession { Ok(response) } - /// Answers a Connect from a peer this session has already accepted. The - /// packet's sequence is remembered and nothing is queued, so a Connect storm - /// cannot grow the reliable queue. The reply is the Accept still awaiting - /// acknowledgement, or an Ack once it was acknowledged. - pub fn answer_repeated_connect( - &mut self, - packet: &CultNetRudpPacket, - now_ms: u64, - ) -> Result { - self.require_connection(packet)?; - if packet.packet_type != CultNetRudpPacketType::Connect { - return Err(anyhow!( - "Expected RUDP connect packet, got {:?}", - packet.packet_type - )); - } - self.apply_acknowledgements(packet); - self.remember_received(packet.sequence); - self.last_received_at_ms = Some(now_ms); - Ok(self - .pending_accept_for_resend(now_ms) - .unwrap_or_else(|| self.create_ack())) - } - pub fn send( &mut self, channel_id: &str, @@ -468,7 +588,7 @@ impl CultNetRudpSession { if options.ordered && !options.reliable { return Err(anyhow!("RUDP ordered delivery requires reliability")); } - self.require_payload_size(payload.len())?; + self.require_payload_size(channel_id, payload.len())?; // Reclaim deadline-passed sends before measuring window capacity, so an // expiring stream cannot be refused for a backlog it no longer owns. self.purge_expired_reliable(options.now_ms); @@ -533,17 +653,38 @@ impl CultNetRudpSession { now_ms: u64, ) -> Result { self.require_connection(packet)?; - self.apply_acknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the + // Accept names it. A late or duplicate one, or one from an earlier + // generation, must not seed the watermark or revive an ended session. + let honours_accept = packet.packet_type == CultNetRudpPacketType::Accept + && self.awaiting_accept + && self + .connect_sequence + .is_some_and(|connect| packet_acknowledges(packet, connect)); + if packet.packet_type == CultNetRudpPacketType::Accept && !honours_accept { + return Ok(CultNetRudpReceiveResult { + delivered: Vec::new(), + ready_to_send: Vec::new(), + reply: None, + pong: false, + pong_payload: Vec::new(), + disconnected: false, + disconnect_reason: Vec::new(), + }); + } + // While this side's Connect awaits its Accept, only the Accept it + // honours retires it: an Ack that names the Connect (a server's reply + // to a repeat or a stale copy) says the server did not start a session. + if honours_accept || !self.awaiting_accept { + self.apply_acknowledgements(packet); + } self.purge_expired_reliable(now_ms); let ready_to_send = self.promote_queued_reliable(now_ms); self.last_received_at_ms = Some(now_ms); - let expected_sequence_if_uninitialized = self - .highest_received_sequence - .map(|sequence| sequence + 1) - .unwrap_or(packet.sequence); - if packet.packet_type == CultNetRudpPacketType::Accept { - self.remember_received(packet.sequence); + if honours_accept { + self.awaiting_accept = false; + self.seed_received(packet.sequence); self.connected = true; return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), @@ -594,13 +735,7 @@ impl CultNetRudpSession { } if packet.packet_type == CultNetRudpPacketType::Disconnect { - // The session is over. What it still owed the peer dies with it, - // and receipts it issued can never become Acknowledged in a later - // session. - self.connected = false; - self.session_scope = Uuid::new_v4(); - self.pending_reliable.clear(); - self.queued_reliable.clear(); + self.end_session(); return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), ready_to_send, @@ -624,6 +759,21 @@ impl CultNetRudpSession { }); } + // Reliable data before the handshake has seeded the watermark has no + // place in the order: refuse it unremembered, so it is not acknowledged + // and the sender retransmits it once the handshake is done. + if packet.reliable && self.received_through.is_none() { + return Ok(CultNetRudpReceiveResult { + delivered: Vec::new(), + ready_to_send, + reply: None, + pong: false, + pong_payload: Vec::new(), + disconnected: false, + disconnect_reason: Vec::new(), + }); + } + let far_ahead = packet.reliable && self.highest_received_sequence.is_some_and(|highest| { packet.sequence > highest @@ -643,12 +793,7 @@ impl CultNetRudpSession { }); } - let duplicate = packet.reliable - && (self.received_sequences.contains(&packet.sequence) - || self.highest_received_sequence.is_some_and(|highest| { - packet.sequence < highest - && highest - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 - })); + let duplicate = packet.reliable && self.was_received(packet.sequence); if packet.reliable { self.remember_received(packet.sequence); } @@ -664,42 +809,36 @@ impl CultNetRudpSession { }); } - let Some((frame, ordered, next_sequence)) = self.reassemble(packet)? else { - return Ok(CultNetRudpReceiveResult { - delivered: Vec::new(), - ready_to_send, - reply: None, - pong: false, - pong_payload: Vec::new(), - disconnected: false, - disconnect_reason: Vec::new(), - }); - }; - let delivered = if ordered { - self.deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized)? - } else if packet.sequenced { - if !self - .latest_sequenced_by_channel - .contains_key(&frame.channel_id) - && self.latest_sequenced_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS - { - return Err(anyhow!("RUDP session has too many sequenced channels")); - } - let latest = self - .latest_sequenced_by_channel - .get(&frame.channel_id) - .copied(); - let newest_sequence = next_sequence.saturating_sub(1); - if latest.is_some_and(|latest| newest_sequence <= latest) { - Vec::new() + let mut delivered = Vec::new(); + if let Some((frame, ordered, next_sequence)) = self.reassemble(packet)? { + if ordered { + self.ordered_held.insert(frame.sequence, frame); + } else if packet.sequenced { + if !self + .latest_sequenced_by_channel + .contains_key(&frame.channel_id) + && self.latest_sequenced_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + { + return Err(anyhow!("RUDP session has too many sequenced channels")); + } + let latest = self + .latest_sequenced_by_channel + .get(&frame.channel_id) + .copied(); + let newest_sequence = next_sequence.saturating_sub(1); + if !latest.is_some_and(|latest| newest_sequence <= latest) { + self.latest_sequenced_by_channel + .insert(frame.channel_id.clone(), newest_sequence); + delivered.push(frame); + } } else { - self.latest_sequenced_by_channel - .insert(frame.channel_id.clone(), newest_sequence); - vec![frame] + delivered.push(frame); } - } else { - vec![frame] - }; + } + // Any reliable packet may have advanced the watermark, a fragment or an + // unordered frame as much as an ordered one, so the drain runs after all + // of them. + delivered.extend(self.drain_ordered()?); Ok(CultNetRudpReceiveResult { delivered, ready_to_send, @@ -748,9 +887,14 @@ impl CultNetRudpSession { } } + /// The acknowledgement for a reliable packet `receive` just handled. What + /// `receive` refused is not acknowledged by name (the ack carries only what + /// was received), so its sender retransmits it. pub fn create_ack_for_received(&mut self, sequence: u32) -> CultNetRudpPacket { let (ack, _) = self.ack_state(); - if ack >= sequence && ack - sequence <= 32 { + if !self.was_received(sequence) { + self.create_ack() + } else if ack >= sequence && ack - sequence <= 32 { self.create_ack() } else { self.create_ack_for(sequence) @@ -769,7 +913,7 @@ impl CultNetRudpSession { } pub fn create_disconnect(&mut self, reason: Vec) -> CultNetRudpPacket { - self.connected = false; + self.end_session(); self.create_packet( CultNetRudpPacketType::Disconnect, "control", @@ -790,11 +934,14 @@ impl CultNetRudpSession { if now_ms.saturating_sub(last_received_at_ms) <= timeout_ms { return false; } - self.connected = false; + self.end_session(); true } pub fn due_resends(&mut self, now_ms: u64) -> Vec { + if let Some(fresh) = self.abandon_unanswered_connect(now_ms) { + return vec![fresh]; + } let mut due = Vec::new(); for pending in self.pending_reliable.values_mut() { if now_ms.saturating_sub(pending.last_sent_at_ms) >= self.resend_delay_ms { @@ -806,6 +953,30 @@ impl CultNetRudpSession { due } + /// A Connect unanswered for `RUDP_CONNECT_ATTEMPT_MS` is replaced, not + /// retransmitted further, by a Connect with the same payload and the + /// abandoned sequence plus the receive window less one. Nothing was ever + /// sent in an unanswered generation, so no sequence issued so far is owed. + /// The jump keeps a late copy of the abandoned Connect inside the new one's + /// stale window, so a server that took the new Connect answers the copy + /// with an Ack instead of restarting; and it leaves the stale window of + /// whatever generation the server holds, unless that generation sits + /// exactly at the jump, and then the next attempt leaves it. The first + /// Connect of a session is the only one drawn at random. + fn abandon_unanswered_connect(&mut self, now_ms: u64) -> Option { + if !self.awaiting_accept + || now_ms.saturating_sub(self.connect_started_at_ms) < RUDP_CONNECT_ATTEMPT_MS + { + return None; + } + let abandoned = self.connect_sequence?; + self.next_sequence = abandoned + .saturating_add(RUDP_RECEIVED_SEQUENCE_WINDOW as u32 - 1) + .min(u32::MAX - 1); + let payload = self.connect_payload.clone(); + self.create_connect(now_ms, payload).ok() + } + fn create_packet( &mut self, packet_type: CultNetRudpPacketType, @@ -970,8 +1141,24 @@ impl CultNetRudpSession { } } + /// The handshake's one act on the watermark: the peer's Connect or Accept + /// is the first sequence of the session, whatever else it has sent. + fn seed_received(&mut self, sequence: u32) { + self.received_through = Some(sequence); + self.remember_received(sequence); + } + fn remember_received(&mut self, sequence: u32) { self.received_sequences.insert(sequence); + if let Some(mut through) = self.received_through { + while let Some(next) = through.checked_add(1) { + if !self.received_sequences.contains(&next) { + break; + } + through = next; + } + self.received_through = Some(through); + } if self .highest_received_sequence .is_none_or(|highest| sequence > highest) @@ -987,6 +1174,22 @@ impl CultNetRudpSession { } } + /// The one duplicate test: true for a sequence in the window that was + /// received, for one below the window, and for one at or before the + /// watermark. The watermark starts at the handshake's seed, so a frame the + /// peer sent in an earlier generation (every sequence it issued is below + /// the Connect that began this one) is a duplicate of something already + /// delivered and is acknowledged, never delivered again. + fn was_received(&self, sequence: u32) -> bool { + self.received_sequences.contains(&sequence) + || self.highest_received_sequence.is_some_and(|highest| { + sequence < highest && highest - sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 + }) + || self + .received_through + .is_some_and(|through| at_or_before(sequence, through)) + } + fn ack_state(&self) -> (u32, u32) { let ack = self.highest_received_sequence.unwrap_or(0); let mut ack_mask = 0_u32; @@ -1003,7 +1206,7 @@ impl CultNetRudpSession { packet: &CultNetRudpPacket, ) -> Result> { if packet.fragment_count == 0 { - self.require_payload_size(packet.payload.len())?; + self.require_payload_size(&packet.channel_id, packet.payload.len())?; return Ok(Some(( CultNetRudpDeliveredFrame { channel_id: packet.channel_id.clone(), @@ -1055,7 +1258,7 @@ impl CultNetRudpSession { .sum::() }) .unwrap_or(0); - if let Err(error) = self.require_payload_size(buffered_bytes + packet.payload.len()) { + if let Err(error) = self.require_payload_size(&packet.channel_id, buffered_bytes + packet.payload.len()) { self.fragment_buffers.remove(&key); return Err(error); } @@ -1113,136 +1316,50 @@ impl CultNetRudpSession { ))) } - fn require_payload_size(&self, payload_bytes: usize) -> Result<()> { - if self - .max_payload_bytes - .is_some_and(|max_payload_bytes| payload_bytes > max_payload_bytes) - { + /// The session's cap, tightened on the media channel to the ceiling every + /// media record is held to (`GAMECULT_MEDIA_MAX_WIRE_BYTES`), so media + /// reassembly never buffers more than a record may be. + fn require_payload_size(&self, channel_id: &str, payload_bytes: usize) -> Result<()> { + let limit = channel_payload_limit(channel_id, self.max_payload_bytes); + if limit.is_some_and(|limit| payload_bytes > limit) { return Err(anyhow!("RUDP payload exceeds max_payload_bytes")); } Ok(()) } - fn ordered_buffered_frames(&self) -> usize { - self.ordered_buffers.values().map(BTreeMap::len).sum() - } - - fn ordered_buffered_bytes(&self) -> usize { - self.ordered_buffers - .values() - .flat_map(BTreeMap::values) - .map(|pending| pending.frame.payload.len()) - .sum() - } - - fn deliver_ordered( - &mut self, - frame: CultNetRudpDeliveredFrame, - next_sequence_after_frame: u32, - expected_sequence_if_uninitialized: u32, - ) -> Result> { - let channel_id = frame.channel_id.clone(); - if !self.ordered_next_sequence_by_channel.contains_key(&channel_id) - && self.ordered_next_sequence_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + /// Delivers, in sequence order, every held ordered frame whose first + /// sequence is at most one past the watermark, then bounds what is still + /// held. Ordered delivery has one owner: the watermark. A frame this call + /// does not deliver waits for the sequence below it, and nothing else + /// releases it. + fn drain_ordered(&mut self) -> Result> { + let Some(through) = self.received_through else { + return Ok(Vec::new()); + }; + let still_held = match through.checked_add(1) { + Some(next) => self.ordered_held.split_off(&next.saturating_add(1)), + None => BTreeMap::new(), + }; + let deliverable = std::mem::replace(&mut self.ordered_held, still_held); + if self.ordered_held.len() > RUDP_MAX_ORDERED_BUFFERED_FRAMES + || self + .ordered_held + .values() + .map(|frame| frame.payload.len()) + .sum::() + > RUDP_MAX_ORDERED_BUFFERED_BYTES { + return Err(anyhow!("RUDP ordered hold buffer is full")); + } + let held_channels: BTreeSet<&str> = self + .ordered_held + .values() + .map(|frame| frame.channel_id.as_str()) + .collect(); + if held_channels.len() > RUDP_MAX_TRACKED_CHANNELS { return Err(anyhow!("RUDP session has too many ordered channels")); } - let mut next = if let Some(next) = self - .ordered_next_sequence_by_channel - .get(&channel_id) - .copied() - { - next - } else { - self.ordered_next_sequence_by_channel.insert( - channel_id.clone(), - expected_sequence_if_uninitialized.min(frame.sequence), - ); - expected_sequence_if_uninitialized.min(frame.sequence) - }; - - while frame.sequence > next - && self.received_sequences.contains(&next) - && !self - .ordered_buffers - .get(&channel_id) - .is_some_and(|buffer| buffer.contains_key(&next)) - { - next = next.saturating_add(1); - self.ordered_next_sequence_by_channel - .insert(channel_id.clone(), next); - } - - if frame.sequence < next { - return Ok(Vec::new()); - } - - if frame.sequence > next { - if self.ordered_buffered_frames() >= RUDP_MAX_ORDERED_BUFFERED_FRAMES - || self.ordered_buffered_bytes() + frame.payload.len() - > RUDP_MAX_ORDERED_BUFFERED_BYTES - { - return Err(anyhow!("RUDP ordered hold buffer is full")); - } - self.ordered_buffers.entry(channel_id).or_default().insert( - frame.sequence, - PendingOrderedFrame { - frame, - next_sequence: next_sequence_after_frame, - }, - ); - return Ok(Vec::new()); - } - - self.ordered_next_sequence_by_channel - .insert(channel_id.clone(), next_sequence_after_frame); - let mut delivered = vec![frame]; - delivered.extend(self.drain_ordered(&channel_id)); - Ok(delivered) - } - - fn drain_ordered(&mut self, channel_id: &str) -> Vec { - let mut delivered = Vec::new(); - loop { - let Some(next) = self - .ordered_next_sequence_by_channel - .get(channel_id) - .copied() - else { - break; - }; - let Some(buffer) = self.ordered_buffers.get_mut(channel_id) else { - break; - }; - let Some(pending) = buffer.remove(&next) else { - break; - }; - delivered.push(pending.frame); - self.ordered_next_sequence_by_channel - .insert(channel_id.to_string(), pending.next_sequence); - self.skip_received_non_channel_sequences(channel_id); - } - delivered - } - - fn skip_received_non_channel_sequences(&mut self, channel_id: &str) { - let Some(mut next) = self - .ordered_next_sequence_by_channel - .get(channel_id) - .copied() - else { - return; - }; - while self.received_sequences.contains(&next) - && !self - .ordered_buffers - .get(channel_id) - .is_some_and(|buffer| buffer.contains_key(&next)) - { - next = next.saturating_add(1); - self.ordered_next_sequence_by_channel - .insert(channel_id.to_string(), next); - } + Ok(deliverable.into_values().collect()) } fn allocate_fragment_id(&mut self) -> u16 { @@ -1280,7 +1397,7 @@ pub struct CultNetRudpSocketTransportOptions { pub mode: CultNetRudpSocketMode, pub remote_addr: Option, pub connection_id: u32, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub transport_id: Option, pub max_payload_bytes: Option, @@ -1307,7 +1424,7 @@ impl CultNetRudpSocketTransportOptions { mode: CultNetRudpSocketMode::Client, remote_addr: Some(remote_addr), connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1326,7 +1443,7 @@ impl CultNetRudpSocketTransportOptions { mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1354,7 +1471,7 @@ pub struct CultNetRudpServerHubOptions { pub runtime_id: String, pub socket: UdpSocket, pub connection_id: u32, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub transport_id: Option, pub max_payload_bytes: Option, @@ -1374,7 +1491,7 @@ impl CultNetRudpServerHubOptions { runtime_id: runtime_id.into(), socket, connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1429,7 +1546,7 @@ struct CultNetRudpServerPeer { pub struct CultNetRudpServerHub { socket: UdpSocket, connection_id: u32, - initial_sequence: u32, + initial_sequence: Option, resend_delay_ms: u64, max_pending_reliable_packets: Option, max_payload_bytes: usize, @@ -1635,9 +1752,9 @@ impl CultNetRudpServerHub { if packet.packet_type == CultNetRudpPacketType::Connect { if let Some(peer) = self.peers.get_mut(&remote_addr) - && peer.context.connect_payload == packet.payload + && peer.session.connect_repeats(&packet) { - let Ok(reply) = peer.session.answer_repeated_connect(&packet, now_ms()) else { + let Ok(reply) = peer.session.accept_connect(&packet, now_ms(), Vec::new()) else { self.end_refusing_peer(remote_addr)?; return Ok(true); }; @@ -1978,7 +2095,11 @@ impl CultNetRudpSocketTransportConnection { } pub fn check_timeout(&mut self, timeout_ms: u64) -> bool { - self.session.check_timeout(now_ms(), timeout_ms) + let timed_out = self.session.check_timeout(now_ms(), timeout_ms); + if timed_out { + self.disconnect_reason = Some(SESSION_TIMED_OUT_REASON.to_vec()); + } + timed_out } pub fn pending_reliable_packet_count(&self) -> usize { @@ -1997,18 +2118,19 @@ impl CultNetRudpSocketTransportConnection { self.socket.set_read_timeout(Some(poll_timeout))?; let deadline = Instant::now() + timeout; - // A flush belongs to the session it started in: a Connect that - // replaces the peer mid-wait forgets the writes being waited on. - let scope = self.session.session_scope(); + // A flush belongs to the generation it started in: whatever ends that + // generation, and whatever begins after it, the writes being waited on + // are gone. + let generation = self.session.generation(); let mut preserved_frames = VecDeque::new(); let result = (|| { loop { // An ended session forgot its unacknowledged writes; reporting // them flushed would be a lie. - if self.session.session_scope() != scope || self.disconnect_reason.is_some() { + if self.session.ended() || self.session.generation() != generation { return Err(anyhow!( "RUDP session ended before its reliable writes were acknowledged: {}", - String::from_utf8_lossy(self.disconnect_reason.as_deref().unwrap_or(b"replaced by a new Connect")) + String::from_utf8_lossy(self.disconnect_reason.as_deref().unwrap_or(b"no reason given")) )); } if self.outstanding_reliable_packet_count() == 0 { @@ -2083,12 +2205,17 @@ impl CultNetRudpSocketTransportConnection { self.stats.packets_dropped += 1; return Ok(true); } + // A Connect from another endpoint is a new client, whatever sequence it + // carries: the endpoint moves and the Connect starts a new generation. + // Only a Connect from the peer's own endpoint can be its repeat. + let mut connect_from_new_endpoint = false; if let Some(expected) = self.remote_addr { if expected != remote_addr { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type == CultNetRudpPacketType::Connect { self.remote_addr = Some(remote_addr); + connect_from_new_endpoint = true; } else { self.stats.packets_dropped += 1; return Ok(true); @@ -2106,8 +2233,10 @@ impl CultNetRudpSocketTransportConnection { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type == CultNetRudpPacketType::Connect { - self.session.reset_peer_state(); self.disconnect_reason = None; + if connect_from_new_endpoint { + self.session.reset_peer_state(); + } let Ok(accept) = self.session.accept_connect(&packet, now_ms(), Vec::new()) else { self.stats.packets_dropped += 1; return Ok(true); @@ -2275,14 +2404,15 @@ where /// Sent to a peer whose session refused one of its packets, and reported to /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; +const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; /// A session cannot admit a Connect when its sequence space starts exhausted or /// its reliable queue holds nothing, so neither is a usable configuration. fn validate_rudp_session_limits( - initial_sequence: u32, + initial_sequence: Option, max_pending_reliable_packets: Option, ) -> Result<()> { - if initial_sequence == u32::MAX { + if initial_sequence == Some(u32::MAX) { return Err(anyhow!( "RUDP initial_sequence must leave room for a reliable packet" )); @@ -2309,6 +2439,18 @@ pub struct RudpTransportProfileOptions { pub media_delivery: Option, } +/// The largest payload a channel accepts: the session-wide cap, tightened on +/// the media channel to the media ceiling. +fn channel_payload_limit(channel_id: &str, session_limit: Option) -> Option { + if channel_id == GAMECULT_MEDIA_CHANNEL { + Some(session_limit.map_or(GAMECULT_MEDIA_MAX_WIRE_BYTES, |limit| { + limit.min(GAMECULT_MEDIA_MAX_WIRE_BYTES) + })) + } else { + session_limit + } +} + pub fn create_rudp_transport_profile( runtime_id: impl Into, options: RudpTransportProfileOptions, @@ -2358,23 +2500,16 @@ pub fn create_rudp_transport_profile( reliable_expire_after_ms: None, }, CultNetTransportChannel { - channel_id: "media".to_string(), + channel_id: GAMECULT_MEDIA_CHANNEL.to_string(), delivery: options .media_delivery .unwrap_or(CultNetTransportDelivery::Reliable), ordering: CultNetTransportOrdering::Unordered, - max_payload_bytes: options.max_payload_bytes, - max_fragment_bytes: options.max_fragment_bytes, - max_pending_reliable_packets: options.max_pending_reliable_packets, - reliable_expire_after_ms: options.media_reliable_expire_after_ms, - }, - // Audio beside lossy video on one session: reliable, but a packet - // older than the media expiry is not worth a retransmit. - CultNetTransportChannel { - channel_id: "audio".to_string(), - delivery: CultNetTransportDelivery::Reliable, - ordering: CultNetTransportOrdering::Unordered, - max_payload_bytes: options.max_payload_bytes, + max_payload_bytes: channel_payload_limit( + GAMECULT_MEDIA_CHANNEL, + options.max_payload_bytes.map(|limit| limit as usize), + ) + .map(|limit| limit as u32), max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: options.max_pending_reliable_packets, reliable_expire_after_ms: options.media_reliable_expire_after_ms, @@ -2544,7 +2679,7 @@ fn channel_send_options( // The one channel whose delivery a caller chooses. Media that has gone // stale is worth dropping, not retransmitting: a reliable media channel // under loss adds load exactly when the link has least to give. - "media" | "audio" => CultNetRudpSendOptions { + "media" => CultNetRudpSendOptions { reliable: matches!(delivery, CultNetTransportDelivery::Reliable), ordered: false, sequenced: false, diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index dfc989e9..686352ba 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -432,12 +432,6 @@ fn rudp_transport_profile_advertises_state_and_realtime_channels() { CultNetTransportOrdering::Unordered, Some(64) ), - ( - "audio", - CultNetTransportDelivery::Reliable, - CultNetTransportOrdering::Unordered, - Some(64) - ), ] ); } @@ -558,13 +552,13 @@ fn rudp_reconnect_loop_consumes_shared_controller() -> Result<()> { fn rudp_session_handshake_acks_reliable_connect_and_accept_packets() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 0x0a0b0c0d, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 50, ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 0x0a0b0c0d, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 50, ..CultNetRudpSessionOptions::default() }); @@ -595,52 +589,19 @@ fn rudp_session_handshake_acks_reliable_connect_and_accept_packets() -> Result<( fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 200, + initial_sequence: Some(200), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 7, - sequence: 1, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 7, - sequence: 2, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -650,20 +611,20 @@ fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Resu let first = sender.send("schema", b"first".to_vec(), options.clone())?; let second = sender.send("schema", b"second".to_vec(), options.clone())?; let third = sender.send("schema", b"third".to_vec(), options)?; - assert_eq!(sender.pending_reliable_sequences(), vec![10, 11, 12]); + assert_eq!(sender.pending_reliable_sequences(), vec![11, 12, 13]); receiver.receive(&first, 0)?; receiver.receive(&third, 0)?; let ack_with_gap = receiver.create_ack(); - assert_eq!(ack_with_gap.ack, 12); - assert_eq!(ack_with_gap.ack_mask, 0b10 | (1 << 9)); + assert_eq!(ack_with_gap.ack, 13); + assert_eq!(ack_with_gap.ack_mask, 0b110); sender.receive(&ack_with_gap, 0)?; - assert_eq!(sender.pending_reliable_sequences(), vec![11]); + assert_eq!(sender.pending_reliable_sequences(), vec![12]); receiver.receive(&second, 0)?; let full_ack = receiver.create_ack(); - assert_eq!(full_ack.ack, 12); - assert_eq!(full_ack.ack_mask, 0b11 | (1 << 9)); + assert_eq!(full_ack.ack, 13); + assert_eq!(full_ack.ack_mask, 0b111); sender.receive(&full_ack, 0)?; assert!(sender.pending_reliable_sequences().is_empty()); Ok(()) @@ -673,28 +634,11 @@ fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Resu fn rudp_session_schedules_reliable_resends_until_acked() -> Result<()> { let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 99, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 99, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); let sent = session.send( "schema", b"payload".to_vec(), @@ -743,12 +687,12 @@ fn rudp_session_schedules_reliable_resends_until_acked() -> Result<()> { fn rudp_session_pings_and_detects_receive_timeout() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 101, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 101, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, b"join".to_vec())?; @@ -774,12 +718,12 @@ fn rudp_session_pings_and_detects_receive_timeout() -> Result<()> { fn rudp_lossy_packets_cannot_create_reliable_ordered_gaps() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 198, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 198, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, Vec::new())?; @@ -830,12 +774,12 @@ fn rudp_lossy_packets_cannot_create_reliable_ordered_gaps() -> Result<()> { fn rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 197, - initial_sequence: 50, + initial_sequence: Some(50), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 197, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -861,24 +805,7 @@ fn rudp_rejects_unreliable_ordered_delivery() -> Result<()> { connection_id: 196, ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 196, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); let error = session .send( "schema", @@ -901,28 +828,11 @@ fn rudp_rejects_unreliable_ordered_delivery() -> Result<()> { fn rudp_session_bounds_pending_reliable_packets_before_enqueue() -> Result<()> { let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 102, - initial_sequence: 1, + initial_sequence: Some(1), max_pending_reliable_packets: Some(2), ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 102, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); session.send( "schema", b"first".to_vec(), @@ -957,28 +867,11 @@ fn rudp_session_bounds_pending_reliable_packets_before_enqueue() -> Result<()> { let mut fragmented = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 103, - initial_sequence: 1, + initial_sequence: Some(1), max_pending_reliable_packets: Some(3), ..CultNetRudpSessionOptions::default() }); - fragmented.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 103, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + fragmented.assume_connected(0); let error = fragmented .send_many( "schema", @@ -1001,50 +894,17 @@ fn rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads_in_ -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 123, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 123, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 123, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 123, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -1082,50 +942,17 @@ fn rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads_in_ fn rudp_session_skips_control_packets_while_ordering_schema_payloads() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 124, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 124, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 124, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 124, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -1164,50 +991,17 @@ fn rudp_session_skips_control_packets_while_ordering_schema_payloads() -> Result fn rudp_session_fragments_and_reassembles_reliable_ordered_payloads() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 456, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 456, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 456, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 456, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let packets = sender.send_many( "schema", @@ -1265,7 +1059,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1282,7 +1076,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1359,7 +1153,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1376,7 +1170,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1404,36 +1198,19 @@ fn rudp_session_advances_large_fragment_sets_through_a_bounded_reliable_window() let connection_id = 0x1020_3042; let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, max_pending_reliable_packets: None, }); - for session in [&mut sender, &mut receiver] { - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id, - sequence: 0, - ack: 0, - ack_mask: 0, - channel_id: "control".into(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - } + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let fragment_count = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; let payload = (0..fragment_count * 8) @@ -1491,7 +1268,7 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1508,7 +1285,7 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1557,6 +1334,71 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { Ok(()) } +fn rudp_client_bound_host(endpoint: &str, bind_host: Option<&str>) -> Result { + let client = CultMesh::create_rudp_client_for_endpoint( + "rust-cultmesh-bind-client", + 0x2030_4060, + endpoint, + CultMeshRudpSocketOptions { + bind_host: bind_host.map(str::to_string), + ..CultMeshRudpSocketOptions::default() + }, + )?; + Ok(client.profile.transports[0] + .host + .clone() + .expect("RUDP client profile advertises its bound host")) +} + +/// A client bound to loopback cannot send to another host on Windows, so a +/// client for a remote endpoint binds the unspecified address of its family. +#[test] +fn cultmesh_rudp_client_for_a_remote_endpoint_binds_the_unspecified_address() -> Result<()> { + assert_eq!(rudp_client_bound_host("rudp://10.77.0.1:17872", None)?, "0.0.0.0"); + assert_eq!(rudp_client_bound_host("rudp://[2001:db8::1]:17872", None)?, "::"); + Ok(()) +} + +#[test] +fn cultmesh_rudp_client_for_a_loopback_endpoint_binds_loopback() -> Result<()> { + assert_eq!(rudp_client_bound_host("rudp://127.0.0.1:17872", None)?, "127.0.0.1"); + assert_eq!(rudp_client_bound_host("rudp://[::1]:17872", None)?, "::1"); + Ok(()) +} + +#[test] +fn cultmesh_rudp_client_binds_an_explicit_bind_host_whatever_the_endpoint() -> Result<()> { + assert_eq!( + rudp_client_bound_host("rudp://10.77.0.1:17872", Some("127.0.0.1"))?, + "127.0.0.1" + ); + assert_eq!( + rudp_client_bound_host("rudp://127.0.0.1:17872", Some("0.0.0.0"))?, + "0.0.0.0" + ); + Ok(()) +} + +#[test] +fn cultmesh_rudp_server_binds_loopback_unless_told_otherwise() -> Result<()> { + let server = CultMesh::create_rudp_server( + "rust-cultmesh-bind-server", + 0x2030_4061, + CultMeshRudpSocketOptions::default(), + )?; + assert_eq!(server.profile.transports[0].host.as_deref(), Some("127.0.0.1")); + let server = CultMesh::create_rudp_server( + "rust-cultmesh-bind-server", + 0x2030_4062, + CultMeshRudpSocketOptions { + bind_host: Some("0.0.0.0".to_string()), + ..CultMeshRudpSocketOptions::default() + }, + )?; + assert_eq!(server.profile.transports[0].host.as_deref(), Some("0.0.0.0")); + Ok(()) +} + #[test] fn cultmesh_facade_creates_rudp_client_from_peer_endpoint() -> Result<()> { let connection_id = 0x2030_4050; @@ -2669,12 +2511,12 @@ fn schema_discovery_round_trips_over_legacy_gamecult_contract_when_inline_schema fn rudp_sequenced_channel_tracking_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 401, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 401, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2709,18 +2551,30 @@ fn rudp_sequenced_channel_tracking_is_bounded() -> Result<()> { fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 402, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 402, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; let accept = receiver.accept_connect(&connect, 1, Vec::new())?; sender.receive(&accept, 2)?; + // Channels are tracked only while they hold a frame, so the bound is on + // channels held behind one missing sequence. + let gap = sender.send( + "gap", + b"lost".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: false, + ..CultNetRudpSendOptions::default() + }, + )?; + sender.receive(&receiver.create_ack_for(gap.sequence), 3)?; let mut error = None; for channel in 0..256u32 { let packet = sender.send( @@ -2733,7 +2587,8 @@ fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { }, )?; match receiver.receive(&packet, 3) { - Ok(_) => { + Ok(result) => { + assert!(result.delivered.is_empty(), "the gap holds every frame"); sender.receive(&receiver.create_ack_for(packet.sequence), 3)?; } Err(refused) => { @@ -2754,12 +2609,12 @@ fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { fn rudp_ordered_hold_buffer_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 403, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 403, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2797,16 +2652,196 @@ fn rudp_ordered_hold_buffer_is_bounded() -> Result<()> { Ok(()) } +fn rudp_ordered_pair() -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(1), + ..CultNetRudpSessionOptions::default() + }); + let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(500), + ..CultNetRudpSessionOptions::default() + }); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.receive(&accept, 0)?; + Ok((client, server)) +} + +fn rudp_send_options(ordered: bool) -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ordered, + ..CultNetRudpSendOptions::default() + } +} + +fn rudp_delivered_names(delivered: &[cultnet_rs::CultNetRudpDeliveredFrame]) -> Vec { + delivered + .iter() + .map(|frame| String::from_utf8(frame.payload.clone()).unwrap()) + .collect() +} + +#[test] +fn rudp_ordered_frame_waits_for_a_gap_filled_by_another_channel() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let o1 = sender.send("schema", b"o1".to_vec(), rudp_send_options(true))?; + let u = sender.send("media", b"u".to_vec(), rudp_send_options(false))?; + let o2 = sender.send("schema", b"o2".to_vec(), rudp_send_options(true))?; + let o3 = sender.send("schema", b"o3".to_vec(), rudp_send_options(true))?; + let o4 = sender.send("schema", b"o4".to_vec(), rudp_send_options(true))?; + + assert_eq!(rudp_delivered_names(&receiver.receive(&o1, 1)?.delivered), ["o1"]); + // `u` is lost, so nothing behind it may be delivered. + assert!(receiver.receive(&o2, 2)?.delivered.is_empty()); + assert!(receiver.receive(&o3, 3)?.delivered.is_empty()); + assert!(receiver.receive(&o4, 4)?.delivered.is_empty()); + // The other channel's packet fills the gap and releases every ordered + // frame behind it in the same call. + assert_eq!( + rudp_delivered_names(&receiver.receive(&u, 5)?.delivered), + ["u", "o2", "o3", "o4"] + ); + Ok(()) +} + +#[test] +fn rudp_two_ordered_channels_release_each_other_in_sequence_order() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let a1 = sender.send("schema", b"A1".to_vec(), rudp_send_options(true))?; + let b1 = sender.send("other", b"B1".to_vec(), rudp_send_options(true))?; + let a2 = sender.send("schema", b"A2".to_vec(), rudp_send_options(true))?; + let a3 = sender.send("schema", b"A3".to_vec(), rudp_send_options(true))?; + + let mut delivered = rudp_delivered_names(&receiver.receive(&a1, 1)?.delivered); + delivered.extend(rudp_delivered_names(&receiver.receive(&a2, 2)?.delivered)); + delivered.extend(rudp_delivered_names(&receiver.receive(&a3, 3)?.delivered)); + assert_eq!(delivered, ["A1"], "B1 is missing, so A2 and A3 wait"); + assert_eq!( + rudp_delivered_names(&receiver.receive(&b1, 4)?.delivered), + ["B1", "A2", "A3"] + ); + Ok(()) +} + +#[test] +fn rudp_channel_first_used_after_other_traffic_loses_nothing() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let c1 = sender.send("late", b"C1".to_vec(), rudp_send_options(true))?; + let c2 = sender.send("late", b"C2".to_vec(), rudp_send_options(true))?; + let x = sender.send("schema", b"X".to_vec(), rudp_send_options(true))?; + + // X, then C2, arrive while C1 is still in flight. + let mut delivered = rudp_delivered_names(&receiver.receive(&x, 1)?.delivered); + delivered.extend(rudp_delivered_names(&receiver.receive(&c2, 2)?.delivered)); + assert!(delivered.is_empty(), "C1 is missing, so nothing is delivered"); + assert_eq!( + rudp_delivered_names(&receiver.receive(&c1, 3)?.delivered), + ["C1", "C2", "X"] + ); + Ok(()) +} + +#[test] +fn rudp_accept_seeds_the_watermark_of_the_connecting_side() -> Result<()> { + let (mut client, mut server) = rudp_ordered_pair()?; + let s1 = server.send("schema", b"s1".to_vec(), rudp_send_options(true))?; + let s2 = server.send("schema", b"s2".to_vec(), rudp_send_options(true))?; + assert!(client.receive(&s2, 1)?.delivered.is_empty()); + assert_eq!( + rudp_delivered_names(&client.receive(&s1, 2)?.delivered), + ["s1", "s2"] + ); + Ok(()) +} + +#[test] +fn rudp_reset_forgets_held_frames_and_reseeds_the_watermark_from_the_next_handshake() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + sender.send("schema", b"s1".to_vec(), rudp_send_options(true))?; + let s2 = sender.send("schema", b"s2".to_vec(), rudp_send_options(true))?; + assert!(receiver.receive(&s2, 1)?.delivered.is_empty()); + + receiver.reset_peer_state(); + let mut next = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(100), + ..CultNetRudpSessionOptions::default() + }); + let accept = receiver.accept_connect(&next.create_connect(0, Vec::new())?, 2, Vec::new())?; + next.receive(&accept, 2)?; + let d1 = next.send("schema", b"d1".to_vec(), rudp_send_options(true))?; + let d2 = next.send("schema", b"d2".to_vec(), rudp_send_options(true))?; + + // The held s2 died with the reset, and the watermark follows the new peer. + assert!(receiver.receive(&d2, 3)?.delivered.is_empty()); + assert_eq!(rudp_delivered_names(&receiver.receive(&d1, 4)?.delivered), ["d1", "d2"]); + Ok(()) +} + +#[test] +fn rudp_large_ordered_frame_without_a_gap_is_never_held_or_refused() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let packets = sender.send_many( + "schema", + vec![7u8; 5 * 1024 * 1024], + rudp_send_options(true), + Some(60_000), + )?; + let mut queue = packets; + let mut delivered = Vec::new(); + while !queue.is_empty() { + let mut next = Vec::new(); + for packet in &queue { + delivered.extend(receiver.receive(packet, 1)?.delivered); + let ack = receiver.create_ack_for_received(packet.sequence); + next.extend(sender.receive(&ack, 1)?.ready_to_send); + } + queue = next; + } + assert_eq!(delivered.len(), 1); + assert_eq!(delivered[0].payload.len(), 5 * 1024 * 1024); + Ok(()) +} + +#[test] +fn rudp_ordered_hold_buffer_is_bounded_in_bytes() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let withheld = sender.send("schema", b"first".to_vec(), rudp_send_options(true))?; + sender.receive(&receiver.create_ack_for(withheld.sequence), 1)?; + + // Four 1 MiB frames fit the 4 MiB hold; the fifth does not. + let mut refused_at = None; + for step in 0..8u32 { + let packet = sender.send("schema", vec![7u8; 1024 * 1024], rudp_send_options(true))?; + match receiver.receive(&packet, 2) { + Ok(result) => { + assert!(result.delivered.is_empty(), "the gap holds every frame"); + sender.receive(&receiver.create_ack_for(packet.sequence), 2)?; + } + Err(refusal) => { + assert!(refusal.to_string().contains("ordered hold buffer is full"), "{refusal}"); + refused_at = Some(step); + break; + } + } + } + assert_eq!(refused_at, Some(4)); + Ok(()) +} + #[test] fn rudp_far_ahead_sequence_cannot_starve_the_session() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 404, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 404, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2855,12 +2890,12 @@ fn rudp_reliable_sequence_exhaustion_is_reported_not_fatal() -> Result<()> { connection_id: 405, // One sequence left: the handshake takes it, so the next send is the // first allocation with nowhere to go. - initial_sequence: u32::MAX - 1, + initial_sequence: Some(u32::MAX - 1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 405, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2924,7 +2959,7 @@ fn tcp_framed_receive_refuses_a_length_beyond_the_advertised_maximum() -> Result fn rudp_reliable_expiry_drops_late_packets_and_leaves_durable_sends_alone() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); @@ -2988,7 +3023,7 @@ fn rudp_reliable_expiry_drops_late_packets_and_leaves_durable_sends_alone() -> R fn rudp_reliable_expiry_also_reclaims_the_send_backlog() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 9, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); @@ -3037,30 +3072,13 @@ fn rudp_reliable_expiry_also_reclaims_the_send_backlog() -> Result<()> { Ok(()) } -fn accept_rudp_session(session: &mut CultNetRudpSession, connection_id: u32) -> Result<()> { - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id, - sequence: 0, - ack: 0, - ack_mask: 0, - channel_id: "control".into(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; +fn accept_rudp_session(session: &mut CultNetRudpSession, _connection_id: u32) -> Result<()> { + session.assume_connected(0); Ok(()) } #[test] -fn rudp_profile_expires_only_the_media_channels() -> Result<()> { +fn rudp_profile_expires_only_the_media_channel() -> Result<()> { let socket = std::net::UdpSocket::bind("127.0.0.1:0")?; let peer = socket.local_addr()?; let options = CultNetRudpSocketTransportOptions::client("expiry-profile", socket, peer, 11); @@ -3074,14 +3092,12 @@ fn rudp_profile_expires_only_the_media_channels() -> Result<()> { .map(|channel| (channel.channel_id.as_str(), channel.reliable_expire_after_ms)) .collect::>(); - for media_channel in ["media", "audio"] { - assert!( - channels.contains(&(media_channel, Some(75))), - "the {media_channel} channel advertises its expiry so a peer can see it: {channels:?}" - ); - } + assert!( + channels.contains(&("media", Some(75))), + "the media channel advertises its expiry so a peer can see it: {channels:?}" + ); for (channel_id, expiry) in &channels { - if !matches!(*channel_id, "media" | "audio") { + if *channel_id != "media" { assert_eq!( *expiry, None, "state channel {channel_id} must never expire a reliable send" @@ -3094,12 +3110,12 @@ fn rudp_profile_expires_only_the_media_channels() -> Result<()> { fn connected_rudp_pair() -> Result<(CultNetRudpSession, CultNetRudpSession)> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 311, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 311, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, Vec::new())?; diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 011ff147..5c633087 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -118,6 +118,46 @@ fn cases() -> Vec { details: None, }, }, + Case { + name: "variant_unsupported_subscription_ended", + // C#: ForVariantUnsupported("variant unsupported here", "sub-1") + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: Some(CultNetErrorDetails { + field: Some("subscriptionId".to_string()), + value: Some("sub-1".to_string()), + as_of: None, + current: None, + }), + }, + }, + Case { + name: "variant_unsupported_no_subscription", + // C#: ForVariantUnsupported("variant unsupported here") - no subscription ended, so Details stays null. + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0", + message: || CultNetMessage::Error { + error: "variant unsupported here".to_string(), + code: Some(CultNetErrorCode::VariantUnsupported), + details: None, + }, + }, + Case { + name: "unowned_schema", + // C#: ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))) + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.".to_string(), + code: Some(CultNetErrorCode::UnownedSchema), + details: Some(CultNetErrorDetails { + field: Some("schemaId".to_string()), + value: Some("some.schema.v1".to_string()), + as_of: None, + current: None, + }), + }, + }, ] } diff --git a/packages/cultnet-rs/tests/impairment/burst-4.toml b/packages/cultnet-rs/tests/impairment/burst-4.toml new file mode 100644 index 00000000..aafe36d0 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/burst-4.toml @@ -0,0 +1,3 @@ +burst_every = 200 +burst_length = 4 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/burst-8.toml b/packages/cultnet-rs/tests/impairment/burst-8.toml new file mode 100644 index 00000000..cc7dd38b --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/burst-8.toml @@ -0,0 +1,3 @@ +burst_every = 200 +burst_length = 8 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/clean.toml b/packages/cultnet-rs/tests/impairment/clean.toml new file mode 100644 index 00000000..ec79f35e --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/clean.toml @@ -0,0 +1,2 @@ +# No impairment. The proxy itself remains in the measured path. +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/delay-jitter.toml b/packages/cultnet-rs/tests/impairment/delay-jitter.toml new file mode 100644 index 00000000..2f62ce3d --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/delay-jitter.toml @@ -0,0 +1,3 @@ +delay_ms = 5 +jitter_ms = 10 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/duplicate.toml b/packages/cultnet-rs/tests/impairment/duplicate.toml new file mode 100644 index 00000000..ad6c761a --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/duplicate.toml @@ -0,0 +1,2 @@ +duplicate_every = 50 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/loss-1pct.toml b/packages/cultnet-rs/tests/impairment/loss-1pct.toml new file mode 100644 index 00000000..e63dacd0 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/loss-1pct.toml @@ -0,0 +1,2 @@ +loss_basis_points = 100 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/loss-3pct.toml b/packages/cultnet-rs/tests/impairment/loss-3pct.toml new file mode 100644 index 00000000..63c4e732 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/loss-3pct.toml @@ -0,0 +1,2 @@ +loss_basis_points = 300 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/loss-5pct.toml b/packages/cultnet-rs/tests/impairment/loss-5pct.toml new file mode 100644 index 00000000..d7cb6d70 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/loss-5pct.toml @@ -0,0 +1,2 @@ +loss_basis_points = 500 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/loss-matrix.txt b/packages/cultnet-rs/tests/impairment/loss-matrix.txt new file mode 100644 index 00000000..cf4f08cc --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/loss-matrix.txt @@ -0,0 +1,37 @@ +profile chunks frames whole/no-fec whole/fec recovered beyond +clean 8 200 200 200 0 0 +clean 16 200 200 200 0 0 +clean 17 200 200 200 0 0 +clean 24 200 200 200 0 0 +clean 32 200 200 200 0 0 +clean 48 200 200 200 0 0 +loss-1pct 8 200 183 200 17 0 +loss-1pct 16 200 170 200 30 0 +loss-1pct 17 200 170 200 31 0 +loss-1pct 24 200 154 200 48 0 +loss-1pct 32 200 144 200 59 0 +loss-1pct 48 200 137 200 75 0 +loss-3pct 8 200 158 199 41 1 +loss-3pct 16 200 126 200 74 0 +loss-3pct 17 200 119 199 90 0 +loss-3pct 24 200 101 200 116 0 +loss-3pct 32 200 78 200 147 0 +loss-3pct 48 200 57 200 208 0 +loss-5pct 8 200 130 197 67 3 +loss-5pct 16 200 93 199 106 1 +loss-5pct 17 200 92 198 132 1 +loss-5pct 24 200 59 198 180 2 +loss-5pct 32 200 41 200 214 0 +loss-5pct 48 200 21 199 318 1 +burst-4 8 200 181 191 10 9 +burst-4 16 200 161 200 39 0 +burst-4 17 200 175 200 46 0 +burst-4 24 200 161 200 68 0 +burst-4 32 200 121 200 118 0 +burst-4 48 200 121 200 177 0 +burst-8 8 200 181 191 10 0 +burst-8 16 200 161 181 20 19 +burst-8 17 200 171 183 20 30 +burst-8 24 200 161 171 19 49 +burst-8 32 200 121 200 118 0 +burst-8 48 200 121 200 197 0 diff --git a/packages/cultnet-rs/tests/impairment/reorder.toml b/packages/cultnet-rs/tests/impairment/reorder.toml new file mode 100644 index 00000000..23c5bf57 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/reorder.toml @@ -0,0 +1,3 @@ +reorder_every = 20 +reorder_delay_ms = 12 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment/stall-250ms.toml b/packages/cultnet-rs/tests/impairment/stall-250ms.toml new file mode 100644 index 00000000..83a30cf7 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment/stall-250ms.toml @@ -0,0 +1,3 @@ +stall_at_ms = 10000 +stall_for_ms = 250 +max_scheduled = 4096 diff --git a/packages/cultnet-rs/tests/impairment_policy.rs b/packages/cultnet-rs/tests/impairment_policy.rs new file mode 100644 index 00000000..2fc612b5 --- /dev/null +++ b/packages/cultnet-rs/tests/impairment_policy.rs @@ -0,0 +1,219 @@ +//! The impairment policy is deterministic, or a loss experiment cannot be +//! replayed. These pin each decision the proxy example makes, against the same +//! `policy.rs` the example compiles. + +#[allow(dead_code)] +#[path = "../examples/cultnet_impair/policy.rs"] +mod policy; + +use std::net::SocketAddr; + +use policy::{Direction, Profile, Scheduler}; + +fn client() -> SocketAddr { + "127.0.0.1:1234".parse().unwrap() +} + +#[test] +fn iid_decisions_are_seed_reproducible() { + let profile = Profile { + loss_basis_points: 2500, + ..Default::default() + }; + let mut a = Scheduler::new(profile.clone(), 42); + let mut b = Scheduler::new(profile, 42); + let left: Vec<_> = (0..100).map(|_| a.decide(0)).collect(); + let right: Vec<_> = (0..100).map(|_| b.decide(0)).collect(); + assert_eq!(left, right); + assert!(left.iter().any(|d| d.drop)); + assert!(left.iter().any(|d| !d.drop)); +} + +#[test] +fn burst_drop_is_exact() { + let profile = Profile { + burst_every: 5, + burst_length: 3, + ..Default::default() + }; + let mut scheduler = Scheduler::new(profile, 1); + let drops: Vec<_> = (1..=9).filter(|_| scheduler.decide(0).drop).collect(); + assert_eq!(drops, vec![5, 6, 7], "a burst starts on the Nth datagram and runs burst_length"); +} + +#[test] +fn duplicate_delay_and_reorder_are_deterministic() { + let profile = Profile { + duplicate_every: 2, + reorder_every: 3, + reorder_delay_ms: 20, + delay_ms: 5, + ..Default::default() + }; + let mut scheduler = Scheduler::new(profile, 1); + assert_eq!(scheduler.decide(0).copies, 1); + assert_eq!(scheduler.decide(0).copies, 2); + let third = scheduler.decide(0); + assert!(third.reordered); + assert_eq!(third.delay_ms, 25); +} + +#[test] +fn stall_drops_only_inside_window() { + let profile = Profile { + stall_at_ms: 100, + stall_for_ms: 50, + ..Default::default() + }; + let mut scheduler = Scheduler::new(profile, 1); + assert!(!scheduler.decide(99).drop); + let stalled = scheduler.decide(100); + assert!(stalled.drop && stalled.stalled); + assert!(scheduler.decide(149).drop); + assert!(!scheduler.decide(150).drop); +} + +#[test] +fn reorder_delay_places_packet_behind_later_packet() { + let profile = Profile { + reorder_every: 2, + reorder_delay_ms: 20, + ..Default::default() + }; + let mut scheduler = Scheduler::new(profile, 1); + scheduler.admit(0, Direction::ClientToUpstream, b"first", client()); + scheduler.admit(0, Direction::ClientToUpstream, b"held", client()); + scheduler.admit(1, Direction::ClientToUpstream, b"later", client()); + assert_eq!(scheduler.pop_due(1).unwrap().bytes, b"first"); + assert_eq!(scheduler.pop_due(1).unwrap().bytes, b"later"); + assert!(scheduler.pop_due(19).is_none()); + assert_eq!(scheduler.pop_due(20).unwrap().bytes, b"held"); +} + +#[test] +fn scheduled_queue_is_bounded_and_counts_overflow() { + let profile = Profile { + delay_ms: 100, + max_scheduled: 2, + ..Default::default() + }; + let mut scheduler = Scheduler::new(profile, 1); + for value in 0..5 { + scheduler.admit(0, Direction::ClientToUpstream, &[value], client()); + } + assert_eq!(scheduler.queue.len(), 2); + assert_eq!(scheduler.stats.max_queue, 2); + assert_eq!(scheduler.stats.queue_overflow, 3); + assert_eq!(scheduler.stats.dropped, 3); +} + +fn profile_path(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/impairment") + .join(format!("{name}.toml")) +} + +#[test] +fn every_committed_profile_loads_with_its_named_meaning() { + let load = |name: &str| Profile::load(&profile_path(name)).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(load("clean"), Profile::default()); + assert_eq!(load("loss-1pct").loss_basis_points, 100); + assert_eq!(load("loss-3pct").loss_basis_points, 300); + assert_eq!(load("loss-5pct").loss_basis_points, 500); + let burst4 = load("burst-4"); + assert_eq!((burst4.burst_every, burst4.burst_length), (200, 4)); + let burst8 = load("burst-8"); + assert_eq!((burst8.burst_every, burst8.burst_length), (200, 8)); + let jitter = load("delay-jitter"); + assert_eq!((jitter.delay_ms, jitter.jitter_ms), (5, 10)); + assert_eq!(load("duplicate").duplicate_every, 50); + let reorder = load("reorder"); + assert_eq!((reorder.reorder_every, reorder.reorder_delay_ms), (20, 12)); + let stall = load("stall-250ms"); + assert_eq!((stall.stall_at_ms, stall.stall_for_ms), (10_000, 250)); +} + +#[test] +fn a_malformed_profile_is_refused_with_its_line() { + let dir = tempfile::tempdir().unwrap(); + let write = |name: &str, text: &str| { + let path = dir.path().join(name); + std::fs::write(&path, text).unwrap(); + Profile::load(&path).unwrap_err() + }; + assert!(write("unknown.toml", "jitter = 3 +").contains("unknown profile key jitter")); + assert!(write("word.toml", "delay_ms = soon +").contains("unsigned integer")); + assert!(write("shape.toml", "delay_ms +").contains("key = value")); + assert!(write("loss.toml", "loss_basis_points = 10001 +").contains("<= 10000")); + assert!(write("queue.toml", "max_scheduled = 0 +").contains("greater than zero")); + assert!(write("line.toml", " + +delay_ms = x +").contains(":3:")); +} + +#[test] +fn admission_counts_what_it_did_to_each_datagram() { + let mut scheduler = Scheduler::new( + Profile { duplicate_every: 2, reorder_every: 3, reorder_delay_ms: 10, ..Default::default() }, + 1, + ); + for value in 0..6 { + scheduler.admit(0, Direction::ClientToUpstream, &[value], client()); + } + // Datagrams 2, 4 and 6 are duplicated, 3 and 6 are reordered (6 is both). + assert_eq!(scheduler.stats.received, 6); + assert_eq!(scheduler.stats.duplicated, 3); + assert_eq!(scheduler.stats.reordered, 2); + assert_eq!(scheduler.queue.len(), 9); + + let mut stalled = Scheduler::new(Profile { stall_at_ms: 0, stall_for_ms: 10, ..Default::default() }, 1); + stalled.admit(5, Direction::UpstreamToClient, b"x", client()); + assert_eq!((stalled.stats.stalled, stalled.stats.dropped, stalled.queue.len()), (1, 1, 0)); +} + +#[test] +fn jitter_is_seeded_uniform_within_its_bound() { + let profile = Profile { delay_ms: 7, jitter_ms: 5, ..Default::default() }; + let mut scheduler = Scheduler::new(profile, 9); + let delays: Vec = (0..300).map(|_| scheduler.decide(0).delay_ms).collect(); + assert!(delays.iter().all(|delay| (7..=12).contains(delay)), "{delays:?}"); + for delay in 7..=12 { + assert!(delays.contains(&delay), "delay {delay} never drawn"); + } +} + +#[test] +fn the_loss_ceiling_is_inclusive() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("all.toml"); + std::fs::write(&path, "loss_basis_points = 10000\n").unwrap(); + let profile = Profile::load(&path).unwrap(); + let mut scheduler = Scheduler::new(profile, 1); + assert!((0..50).all(|_| scheduler.decide(0).drop), "10000 basis points drops everything"); +} + +/// The seeded stream is part of the contract: the same seed must drop the same +/// datagrams on every run, on every host. 100,000 draws of xorshift64 from seed +/// 42, taken modulo 10,000 and compared against 100, give 1,008. +#[test] +fn seeded_loss_drops_the_exact_datagrams_the_stream_dictates() { + let mut scheduler = Scheduler::new(Profile { loss_basis_points: 100, ..Default::default() }, 42); + let dropped = (0..100_000).filter(|_| scheduler.decide(0).drop).count(); + assert_eq!(dropped, 1_008); +} + +#[test] +fn datagrams_due_together_leave_in_arrival_order() { + let mut scheduler = Scheduler::new(Profile { delay_ms: 5, ..Default::default() }, 1); + for value in 0..4_u8 { + scheduler.admit(0, Direction::ClientToUpstream, &[value], client()); + } + let order: Vec = std::iter::from_fn(|| scheduler.pop_due(5)).map(|item| item.bytes[0]).collect(); + assert_eq!(order, vec![0, 1, 2, 3]); +} diff --git a/packages/cultnet-rs/tests/media_channel_delivery.rs b/packages/cultnet-rs/tests/media_channel_delivery.rs index 499be0d3..d902e120 100644 --- a/packages/cultnet-rs/tests/media_channel_delivery.rs +++ b/packages/cultnet-rs/tests/media_channel_delivery.rs @@ -90,3 +90,121 @@ fn the_other_channels_keep_their_fixed_delivery() { assert_eq!(delivery_of("schema"), CultNetTransportDelivery::Reliable); assert_eq!(delivery_of("realtime"), CultNetTransportDelivery::Unreliable); } + +// --------------------------------------------------------------------------- +// What the media channel puts on the wire +// --------------------------------------------------------------------------- + +use std::thread::sleep; +use std::time::Duration; + +use cultnet_rs::{ + CultNetRudpPacket, CultNetRudpSession, CultNetRudpSessionOptions, decode_rudp_packet, + encode_rudp_packet, +}; + +/// A connected client transport and the raw socket standing in for its peer. +struct Link { + client: CultNetRudpSocketTransportConnection, + peer: UdpSocket, +} + +impl Link { + fn open(delivery: Option, expire_after_ms: Option) -> Self { + let peer = UdpSocket::bind("127.0.0.1:0").expect("binds"); + peer.set_read_timeout(Some(Duration::from_millis(500))).unwrap(); + let socket = UdpSocket::bind("127.0.0.1:0").expect("binds"); + socket.set_read_timeout(Some(Duration::from_millis(300))).unwrap(); + let mut options = CultNetRudpSocketTransportOptions::client( + "test-media", + socket, + peer.local_addr().unwrap(), + 7, + ); + options.media_delivery = delivery; + options.media_reliable_expire_after_ms = expire_after_ms; + options.resend_delay_ms = 10; + let mut client = CultNetRudpSocketTransportConnection::new(options).expect("opens"); + + client.connect(Vec::new()).expect("sends connect"); + let mut wire = vec![0_u8; 2_048]; + let (received, client_addr) = peer.recv_from(&mut wire).expect("the connect arrives"); + let connect = decode_rudp_packet(&wire[..received]).expect("decodes"); + let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + ..CultNetRudpSessionOptions::default() + }); + let accept = server.accept_connect(&connect, 0, Vec::new()).expect("accepts"); + peer.send_to(&encode_rudp_packet(&accept).unwrap(), client_addr).unwrap(); + client.receive_once().expect("takes the accept"); + assert!(client.connected()); + let mut link = Self { client, peer }; + link.datagrams(Duration::from_millis(100)); + link + } + + /// Everything the peer receives within `window`. + fn datagrams(&mut self, window: Duration) -> Vec { + self.peer.set_read_timeout(Some(window)).unwrap(); + let mut packets = Vec::new(); + let mut wire = vec![0_u8; 2_048]; + while let Ok((received, _)) = self.peer.recv_from(&mut wire) { + packets.push(decode_rudp_packet(&wire[..received]).expect("decodes")); + } + packets + } + + fn send_media(&mut self, window: Duration) -> Vec { + self.client.send("media", vec![1, 2, 3]).expect("sends"); + self.datagrams(window) + } +} + +fn media_only(packets: Vec) -> Vec { + packets.into_iter().filter(|packet| packet.channel_id == "media").collect() +} + +/// The channel the profile describes as reliable is sent reliable, unordered +/// and unsequenced; asking for an unreliable lane sends it unreliable. +#[test] +fn the_media_channel_is_sent_with_the_delivery_the_profile_advertises() { + let mut reliable = Link::open(None, None); + let sent = media_only(reliable.send_media(Duration::from_millis(100))); + assert_eq!(sent.len(), 1); + assert!(sent[0].reliable, "the default media lane is reliable"); + assert!(!sent[0].ordered && !sent[0].sequenced, "media is neither ordered nor sequenced"); + + let mut unreliable = Link::open(Some(CultNetTransportDelivery::Unreliable), None); + let sent = media_only(unreliable.send_media(Duration::from_millis(100))); + assert_eq!(sent.len(), 1); + assert!(!sent[0].reliable, "an unreliable media lane is not retransmitted"); + assert!(!sent[0].ordered && !sent[0].sequenced); +} + +/// A reliable media packet with an expiry is dropped once it has passed, and +/// one without is retransmitted until acknowledged. The session reclaims +/// expired packets on the next send, so the probe sends a second packet. +fn first_packet_is_resent_after_a_later_send(expire_after_ms: Option) -> bool { + let mut link = Link::open(None, expire_after_ms); + let first = media_only(link.send_media(Duration::from_millis(50))); + assert_eq!(first.len(), 1); + sleep(Duration::from_millis(150)); + assert_eq!(media_only(link.send_media(Duration::from_millis(50))).len(), 1); + sleep(Duration::from_millis(15)); + link.client.poll_resends().unwrap(); + media_only(link.datagrams(Duration::from_millis(100))) + .iter() + .any(|packet| packet.sequence == first[0].sequence) +} + +#[test] +fn the_media_channels_expiry_is_applied_to_its_packets() { + assert!( + !first_packet_is_resent_after_a_later_send(Some(20)), + "an expired media packet is dropped, not retransmitted" + ); + assert!( + first_packet_is_resent_after_a_later_send(None), + "without an expiry the unacknowledged packet is retransmitted" + ); +} diff --git a/packages/cultnet-rs/tests/media_fec.rs b/packages/cultnet-rs/tests/media_fec.rs new file mode 100644 index 00000000..4af9f243 --- /dev/null +++ b/packages/cultnet-rs/tests/media_fec.rs @@ -0,0 +1,1143 @@ +//! The media FEC codec's guarantees, proved against its public surface. +//! +//! Every test drives `protect_*` and `recover_*` the way a producer and a +//! consumer would, and asserts what a consumer is entitled to rely on: what is +//! recovered is exactly what was sent, what cannot be recovered is reported and +//! never guessed, and a shard always fits a datagram. + +use cultnet_rs::{ + GameCultMediaAudioPacketRecord, GameCultMediaAudioParityShardRecord, + GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, + GAMECULT_MEDIA_MAX_WIRE_BYTES, GameCultMediaWireRecord, MEDIA_FEC_MAX_WIRE_BYTES, + MEDIA_FEC_SCHEME_RS_GF256_V1, MediaFecError, MediaFecPolicy, MediaWireProvenance, + decode_media_wire_record, encode_media_wire_record, protect_audio_block, protect_video_frame, + recover_audio_block, recover_video_block, +}; + +const STANDARD: MediaFecPolicy = MediaFecPolicy::STANDARD; + +/// Deterministic, non-constant bytes so a shard mix-up cannot pass by accident. +fn bytes(seed: u32, len: usize) -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect() +} + +/// One frame of `chunk_count` chunks: every chunk `chunk_bytes` long except the +/// last, which is `last_bytes`. +fn frame( + frame_id: u64, + chunk_count: u16, + chunk_bytes: usize, + last_bytes: usize, +) -> Vec { + (0..chunk_count) + .map(|index| GameCultMediaVideoAccessUnitRecord { + stream_id: "muninn.raven.av.rudp".to_string(), + session_id: "session-1".to_string(), + frame_id, + codec: "h264".to_string(), + pts_ticks: 90_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + keyframe: frame_id % 2 == 0, + dependency_frame_id: (frame_id % 2 == 1).then(|| frame_id - 1), + deadline_ticks: 90_000 + 22_500, + chunk_index: index, + chunk_count, + payload: bytes( + (frame_id as u32).wrapping_mul(1_000).wrapping_add(u32::from(index)), + if index + 1 == chunk_count { last_bytes } else { chunk_bytes }, + ), + }) + .collect() +} + +fn audio_block(base_packet_id: u64, payload_bytes: usize) -> Vec { + (0..4_u64) + .map(|index| GameCultMediaAudioPacketRecord { + stream_id: "muninn.raven.av.rudp".to_string(), + session_id: "session-1".to_string(), + packet_id: base_packet_id + index, + codec: "opus".to_string(), + pts_ticks: 48_000 + 960 * index as i64, + duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 60_000 + 960 * index as i64, + payload: bytes((base_packet_id as u32).wrapping_mul(10).wrapping_add(index as u32), payload_bytes), + }) + .collect() +} + +fn split( + records: Vec, +) -> (Vec, Vec) { + let mut data = Vec::new(); + let mut parity = Vec::new(); + for record in records { + match record { + GameCultMediaWireRecord::Video(record) => data.push(record), + GameCultMediaWireRecord::VideoParity(record) => parity.push(record), + other => panic!("a video frame produced {}", other.schema_id()), + } + } + (data, parity) +} + +/// The block a wire record belongs to, from its own fields. +fn block_of(record: &GameCultMediaWireRecord, sizes: &[u16]) -> usize { + match record { + GameCultMediaWireRecord::Video(chunk) => { + let mut start = 0_u16; + for (block, size) in sizes.iter().enumerate() { + if chunk.chunk_index < start + size { + return block; + } + start += size; + } + panic!("chunk {} is outside every block", chunk.chunk_index) + } + GameCultMediaWireRecord::VideoParity(parity) => usize::from(parity.block_index), + other => panic!("not a video record: {}", other.schema_id()), + } +} + +// --------------------------------------------------------------------------- +// Recovery: the MDS guarantee, exhaustively +// --------------------------------------------------------------------------- + +#[test] +fn the_policy_gives_the_ruled_block_geometry() { + // m = max(2, ceil(k / 4)), written independently of the policy. + for k in 1..=16_u16 { + let expected = std::cmp::max(2, (k + 3) / 4); + assert_eq!(STANDARD.video_parity_shards(k), expected, "k = {k}"); + } + assert_eq!(STANDARD.audio_data_shards, 4); + assert_eq!(STANDARD.audio_parity_shards, 2); +} + +#[test] +fn a_frame_splits_into_ceil_n_over_16_near_equal_blocks() { + for n in 1..=200_u16 { + let sizes = STANDARD.video_block_sizes(n); + assert_eq!(sizes.len(), usize::from(n.div_ceil(16)), "n = {n}"); + assert_eq!(sizes.iter().sum::(), n, "n = {n}"); + assert!(sizes.iter().all(|size| (1..=16).contains(size)), "n = {n}: {sizes:?}"); + let (widest, narrowest) = (sizes.iter().max().unwrap(), sizes.iter().min().unwrap()); + assert!(widest - narrowest <= 1, "n = {n}: {sizes:?}"); + assert!(sizes.windows(2).all(|pair| pair[0] >= pair[1]), "larger blocks first: {sizes:?}"); + } +} + +/// Erases every pattern of at most `m` records from a `k`-chunk frame's wire +/// output and requires the data back, byte for byte, then requires every +/// `m + 1` pattern to be refused. +fn assert_every_erasure_pattern_behaves(k: u16) { + let original = frame(u64::from(k), k, 37, 21); + let records = protect_video_frame(&original, &STANDARD, provenance()).expect("protects"); + let m = usize::from(STANDARD.video_parity_shards(k)); + let total = usize::from(k) + m; + assert_eq!(records.len(), total, "one block: k data and m parity records"); + + for erased in 0_u32..(1 << total) { + let erasures = erased.count_ones() as usize; + if erasures > m + 1 { + continue; + } + let mut data = Vec::new(); + let mut parity = Vec::new(); + for (slot, record) in records.iter().enumerate() { + if erased & (1 << slot) != 0 { + continue; + } + match record { + GameCultMediaWireRecord::Video(chunk) => data.push(chunk.clone()), + GameCultMediaWireRecord::VideoParity(shard) => parity.push(shard.clone()), + _ => unreachable!(), + } + } + if parity.is_empty() { + continue; // a receiver holding no parity has nothing to recover with + } + let outcome = recover_video_block(&parity, &data); + if erasures <= m { + let recovered = outcome.unwrap_or_else(|error| { + panic!("k={k} m={m} erased={erased:#b}: {error}"); + }); + data.extend(recovered); + data.sort_by_key(|chunk| chunk.chunk_index); + assert_eq!(data, original, "k={k} m={m} erased={erased:#b}"); + } else { + assert!( + matches!(outcome, Err(MediaFecError::BeyondRepair { .. })), + "k={k} m={m} erased={erased:#b}: {outcome:?} must be beyond repair, never wrong bytes" + ); + } + } +} + +#[test] +fn every_le_m_erasure_recovers_exactly_and_m_plus_one_is_refused_for_every_video_k() { + for k in 1..=16 { + assert_every_erasure_pattern_behaves(k); + } +} + +#[test] +fn every_le_m_erasure_recovers_exactly_and_m_plus_one_is_refused_for_audio_4_plus_2() { + let original = audio_block(100, 41); + let parity = protect_audio_block(&original, &STANDARD, provenance()).expect("protects"); + assert_eq!(parity.len(), 2); + for erased in 0_u32..(1 << 6) { + let erasures = erased.count_ones(); + let data: Vec<_> = (0..4) + .filter(|slot| erased & (1 << slot) == 0) + .map(|slot| original[slot].clone()) + .collect(); + let present_parity: Vec<_> = (0..2) + .filter(|slot| erased & (1 << (4 + slot)) == 0) + .map(|slot| parity[slot].clone()) + .collect(); + if present_parity.is_empty() { + continue; + } + let outcome = recover_audio_block(&present_parity, &data); + if erasures <= 2 { + let mut all = data.clone(); + let block_deadline = original.iter().map(|p| p.deadline_ticks).max().unwrap(); + for mut packet in outcome.unwrap_or_else(|error| panic!("erased={erased:#b}: {error}")) { + // A recovered packet carries the block's deadline, not its own. + assert_eq!(packet.deadline_ticks, block_deadline, "erased={erased:#b}"); + packet.deadline_ticks = original[(packet.packet_id - 100) as usize].deadline_ticks; + all.push(packet); + } + all.sort_by_key(|packet| packet.packet_id); + assert_eq!(all, original, "erased={erased:#b}"); + } else { + assert!( + matches!(outcome, Err(MediaFecError::BeyondRepair { .. })), + "erased={erased:#b}: {outcome:?}" + ); + } + } +} + +#[test] +fn recovery_reports_how_short_the_block_is() { + let original = frame(9, 8, 37, 37); + let (data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + // 8 data + 2 parity; keep 6 data and 1 parity: 7 present, 8 needed. + let outcome = recover_video_block(&parity[..1], &data[..6]); + assert_eq!( + outcome, + Err(MediaFecError::BeyondRepair { shards_present: 7, shards_needed: 8 }) + ); +} + +#[test] +fn a_recovered_short_final_chunk_has_its_true_length() { + let original = frame(4, 5, 100, 7); + let (mut data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + let lost = data.pop().unwrap(); + assert_eq!(lost.payload.len(), 7); + let recovered = recover_video_block(&parity, &data).unwrap(); + assert_eq!(recovered, vec![lost]); +} + +#[test] +fn duplicate_shards_neither_count_twice_nor_break_recovery() { + let original = frame(5, 4, 50, 50); + let (mut data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + let lost = data.remove(1); + data.push(data[0].clone()); + let mut parity_twice = parity.clone(); + parity_twice.extend(parity); + assert_eq!(recover_video_block(&parity_twice, &data).unwrap(), vec![lost]); +} + +#[test] +fn a_complete_block_recovers_nothing() { + let original = frame(6, 4, 50, 50); + let (data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + assert_eq!(recover_video_block(&parity[..1], &data).unwrap(), Vec::new()); +} + +#[test] +fn recovery_refuses_records_that_are_not_one_block() { + let original = frame(7, 32, 40, 40); + let (data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + let block_zero: Vec<_> = parity.iter().filter(|p| p.block_index == 0).cloned().collect(); + let block_one: Vec<_> = parity.iter().filter(|p| p.block_index == 1).cloned().collect(); + assert_eq!(block_zero.len(), 4); + + let mixed = vec![block_zero[0].clone(), block_one[0].clone()]; + assert!(matches!(recover_video_block(&mixed, &[]), Err(MediaFecError::Invalid(_)))); + + // A chunk from block one handed to block zero's parity. + // The first chunk of block one, exactly one past block zero's last. + let stray = data.iter().find(|chunk| chunk.chunk_index == 16).unwrap().clone(); + assert!(matches!(recover_video_block(&block_zero, &[stray]), Err(MediaFecError::Invalid(_)))); + + // A chunk whose length disagrees with the block's. + let mut wrong_length = data[0].clone(); + wrong_length.payload.push(0); + assert!(matches!( + recover_video_block(&block_zero, &[wrong_length]), + Err(MediaFecError::Invalid(_)) + )); + + assert!(matches!(recover_video_block(&[], &data), Err(MediaFecError::Invalid(_)))); + + // Another frame's chunk. + let mut other_frame = data.iter().find(|chunk| chunk.chunk_index == 3).unwrap().clone(); + other_frame.frame_id += 1; + assert!(matches!(recover_video_block(&block_zero, &[other_frame]), Err(MediaFecError::Invalid(_)))); +} + +#[test] +fn a_final_chunk_of_the_wrong_length_is_refused() { + let original = frame(8, 5, 100, 7); + let (data, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + let mut last = data.iter().find(|chunk| chunk.chunk_index == 4).unwrap().clone(); + last.payload.push(0); + assert!(matches!(recover_video_block(&parity, &[last]), Err(MediaFecError::Invalid(_)))); +} + +#[test] +fn within_a_block_data_and_parity_alternate() { + let records = protect_video_frame(&frame(9, 4, 20, 20), &STANDARD, provenance()).unwrap(); + let kinds: String = records + .iter() + .map(|record| match record { + GameCultMediaWireRecord::Video(_) => 'D', + GameCultMediaWireRecord::VideoParity(_) => 'P', + _ => '?', + }) + .collect(); + assert_eq!(kinds, "DPDPDD", "k = 4, m = 2: the longer list finishes the lane"); +} + +#[test] +fn audio_recovery_refuses_packets_that_are_not_from_the_block() { + let original = audio_block(100, 20); + let parity = protect_audio_block(&original, &STANDARD, provenance()).unwrap(); + let next_block = audio_block(104, 20); + // One past the block's last packet, and one before its first. + for stray in [next_block[0].clone(), audio_block(96, 20)[3].clone()] { + assert!(matches!( + recover_audio_block(&parity, &[stray]), + Err(MediaFecError::Invalid(_)) + )); + } + let mut other_session = original[0].clone(); + other_session.session_id = "session-2".to_string(); + assert!(matches!(recover_audio_block(&parity, &[other_session]), Err(MediaFecError::Invalid(_)))); + let mut wrong_length = original[0].clone(); + wrong_length.payload.push(0); + assert!(matches!(recover_audio_block(&parity, &[wrong_length]), Err(MediaFecError::Invalid(_)))); + assert!(matches!(recover_audio_block(&[], &original), Err(MediaFecError::Invalid(_)))); + + let mut mixed = parity.clone(); + mixed[1].base_packet_id += 1; + assert!(matches!(recover_audio_block(&mixed, &original[..3]), Err(MediaFecError::Invalid(_)))); +} + +#[test] +fn a_frame_of_several_blocks_recovers_each_block_from_its_own_parity() { + let original = frame(11, 40, 60, 33); + let records = protect_video_frame(&original, &STANDARD, provenance()).unwrap(); + let sizes = STANDARD.video_block_sizes(40); + assert_eq!(sizes, vec![14, 13, 13]); + let (data, parity) = split(records); + + // Lose the first two data chunks of every block. + let mut lost_indexes = Vec::new(); + let mut start = 0_u16; + for size in &sizes { + lost_indexes.extend([start, start + 1]); + start += size; + } + let held: Vec<_> = data.iter().filter(|c| !lost_indexes.contains(&c.chunk_index)).cloned().collect(); + let mut all = held.clone(); + for block in 0..3_u16 { + let block_parity: Vec<_> = parity.iter().filter(|p| p.block_index == block).cloned().collect(); + let lo = sizes[..usize::from(block)].iter().sum::(); + let block_data: Vec<_> = held + .iter() + .filter(|c| c.chunk_index >= lo && c.chunk_index < lo + sizes[usize::from(block)]) + .cloned() + .collect(); + all.extend(recover_video_block(&block_parity, &block_data).unwrap()); + } + all.sort_by_key(|chunk| chunk.chunk_index); + assert_eq!(all, original); +} + +// --------------------------------------------------------------------------- +// Known answers: the scheme is pinned, not the library +// --------------------------------------------------------------------------- + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Four 37-byte audio payloads, `payload[i][j] = (i * 37 + j * 7 + 3) mod 256`. +/// If these bytes ever change, `rs-gf256-v1` changed meaning: the matrix, the +/// field polynomial, or the shard order. Bump the scheme id, do not edit them. +#[test] +fn known_answer_audio_parity_bytes() { + let mut block = audio_block(1, 37); + for (i, packet) in block.iter_mut().enumerate() { + packet.payload = (0..37).map(|j| (i * 37 + j * 7 + 3) as u8).collect(); + } + let parity = protect_audio_block(&block, &STANDARD, provenance()).unwrap(); + assert_eq!(parity[0].fec_scheme, "rs-gf256-v1"); + assert_eq!( + hex(&parity[0].payload), + "f246399711f2ecefa4147ae3856317be83b9a87c0ed13bc552918473bc08e869a91dd98356", + "parity 0" + ); + assert_eq!( + hex(&parity[1].payload), + "a1532c08e588cc05da6175c61b09a704f7d6c7597d76fbef76435656bf65e2331def91dcbf", + "parity 1" + ); +} + +/// The video path pads a short final chunk with zeros before computing parity. +/// Chunks of 5, 5 and 3 bytes, `payload[i][j] = (i * 11 + j * 3 + 1)`. +#[test] +fn known_answer_video_parity_bytes_pad_a_short_final_chunk_with_zeros() { + let mut chunks = frame(2, 3, 5, 3); + for chunk in &mut chunks { + let len = chunk.payload.len(); + chunk.payload = (0..len).map(|j| (usize::from(chunk.chunk_index) * 11 + j * 3 + 1) as u8).collect(); + } + let (_, parity) = split(protect_video_frame(&chunks, &STANDARD, provenance()).unwrap()); + assert_eq!(parity.len(), 2); + assert_eq!(hex(&parity[0].payload), "1a11081f15", "parity 0"); + assert_eq!(hex(&parity[1].payload), "1d18f3ce8b", "parity 1"); + assert_eq!(parity[0].shard_payload_bytes, 5); + assert_eq!(parity[0].last_chunk_payload_bytes, 3); +} + +#[test] +fn parity_is_a_pure_function_of_its_inputs() { + let original = frame(3, 20, 200, 90); + assert_eq!( + protect_video_frame(&original, &STANDARD, provenance()).unwrap(), + protect_video_frame(&original, &STANDARD, provenance()).unwrap() + ); + let block = audio_block(8, 64); + assert_eq!( + protect_audio_block(&block, &STANDARD, provenance()).unwrap(), + protect_audio_block(&block, &STANDARD, provenance()).unwrap() + ); +} + +// --------------------------------------------------------------------------- +// One datagram +// --------------------------------------------------------------------------- + +fn provenance() -> MediaWireProvenance<'static> { + MediaWireProvenance { + stored_at: "unix:1700000000000", + runtime_id: "raven-muninn-primary", + role: "muninn.media", + producer: "muninn", + } +} + +#[test] +fn the_datagram_budget_is_the_rudp_fragment_the_media_channel_leaves() { + // 1,472 bytes of UDP payload, less the 36-byte RUDP header and the 5-byte + // channel name "media". + assert_eq!(MEDIA_FEC_MAX_WIRE_BYTES, 1_431); +} + +/// A provenance whose every field is 32 bytes, the widest the tests claim. +fn widest_provenance() -> MediaWireProvenance<'static> { + let field = |prefix: &str| -> &'static str { Box::leak(long(prefix).into_boxed_str()) }; + MediaWireProvenance { + stored_at: field("unix:"), + runtime_id: field("runtime-"), + role: field("role-"), + producer: field("producer-"), + } +} + +fn long(prefix: &str) -> String { + format!("{prefix}{}", "x".repeat(32 - prefix.len())) +} + +/// Twenty chunks (two blocks) of an odd frame with a dependency, every field +/// at the width MessagePack spends the most bytes on. +fn widest_frame(payload_bytes: usize) -> Vec { + let mut chunks = frame(u64::MAX, 20, payload_bytes, payload_bytes); + for chunk in &mut chunks { + chunk.stream_id = long("stream-"); + chunk.session_id = long("session-"); + chunk.codec = long("codec-"); + chunk.pts_ticks = i64::MIN; + chunk.deadline_ticks = i64::MAX; + chunk.duration_ticks = u32::MAX; + chunk.timebase_num = u32::MAX; + chunk.timebase_den = u32::MAX; + assert!(chunk.dependency_frame_id.is_some() && !chunk.keyframe); + } + chunks +} + +fn widest_audio(payload_bytes: usize) -> Vec { + let mut packets = audio_block(u64::MAX - 3, payload_bytes); + for (index, packet) in packets.iter_mut().enumerate() { + packet.stream_id = long("stream-"); + packet.session_id = long("session-"); + packet.codec = long("codec-"); + packet.duration_ticks = u32::MAX; + packet.timebase_num = u32::MAX; + packet.timebase_den = u32::MAX; + packet.pts_ticks = i64::MIN + i64::from(u32::MAX) * index as i64; + packet.deadline_ticks = i64::MAX; + } + packets +} + +/// Fitting is monotonic in the payload, so the largest payload that fits is a +/// partition point. +fn largest_fitting_payload(fits: impl Fn(usize) -> bool) -> usize { + let payloads: Vec = (1..=MEDIA_FEC_MAX_WIRE_BYTES).collect(); + let count = payloads.partition_point(|&payload| fits(payload)); + assert!(count > 0, "no payload fits"); + payloads[count - 1] +} + +fn wire_len(record: &GameCultMediaWireRecord, provenance: MediaWireProvenance<'_>) -> usize { + encode_media_wire_record(record, provenance).unwrap().len() +} + +fn is_over_budget(error: &MediaFecError) -> bool { + matches!(error, MediaFecError::Invalid(message) if message.contains("datagram budget")) +} + +/// The invariant is "an encoded record fits one datagram", enforced where the +/// record is produced. At the widest legal identifiers the largest payload that +/// fits is found by search, the record it produces is exactly the budget, and +/// one byte more is refused. +#[test] +fn a_video_frame_is_protected_only_if_every_record_fits_the_datagram_budget() { + let widest = widest_provenance(); + let fits = |payload: usize| protect_video_frame(&widest_frame(payload), &STANDARD, widest); + let largest_payload = largest_fitting_payload(|payload| fits(payload).is_ok()); + assert!( + largest_payload >= 600, + "maximum-width identifiers must still leave room for a useful chunk, got {largest_payload}" + ); + + let records = fits(largest_payload).unwrap(); + let largest = records.iter().map(|record| wire_len(record, widest)).max().unwrap(); + assert_eq!(largest, MEDIA_FEC_MAX_WIRE_BYTES, "the budget is used to the byte, not approximated"); + + let error = fits(largest_payload + 1).unwrap_err(); + assert!(is_over_budget(&error), "{error}"); + + // The budget is the policy's, and it is inclusive. + let exact = MediaFecPolicy { max_wire_bytes: largest, ..STANDARD }; + assert!(protect_video_frame(&widest_frame(largest_payload), &exact, widest).is_ok()); + let one_short = MediaFecPolicy { max_wire_bytes: largest - 1, ..STANDARD }; + let error = protect_video_frame(&widest_frame(largest_payload), &one_short, widest).unwrap_err(); + assert!(is_over_budget(&error), "{error}"); +} + +#[test] +fn an_audio_block_is_protected_only_if_every_record_fits_the_datagram_budget() { + let widest = widest_provenance(); + let fits = |payload: usize| protect_audio_block(&widest_audio(payload), &STANDARD, widest); + let largest_payload = largest_fitting_payload(|payload| fits(payload).is_ok()); + assert!(largest_payload >= 600, "got {largest_payload}"); + + let packets = widest_audio(largest_payload); + let parity = fits(largest_payload).unwrap(); + let largest_parity = parity + .iter() + .map(|shard| wire_len(&GameCultMediaWireRecord::AudioParity(shard.clone()), widest)) + .max() + .unwrap(); + assert_eq!(largest_parity, MEDIA_FEC_MAX_WIRE_BYTES); + // The parity is the largest record of the block, which is why only parity + // is measured. + for packet in &packets { + assert!(wire_len(&GameCultMediaWireRecord::Audio(packet.clone()), widest) < largest_parity); + } + + let error = fits(largest_payload + 1).unwrap_err(); + assert!(is_over_budget(&error), "{error}"); + let one_short = MediaFecPolicy { max_wire_bytes: largest_parity - 1, ..STANDARD }; + let error = protect_audio_block(&packets, &one_short, widest).unwrap_err(); + assert!(is_over_budget(&error), "{error}"); +} + +/// A path that carries less than the IPv4 default passes a smaller budget, and +/// a frame protected under the default is refused under it. +#[test] +fn a_smaller_datagram_budget_refuses_what_the_default_admits() { + let chunks = frame(2, 20, 700, 700); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_ok()); + let tunnel = MediaFecPolicy { max_wire_bytes: 1_211, ..STANDARD }; + let error = protect_video_frame(&chunks, &tunnel, provenance()).unwrap_err(); + assert!(is_over_budget(&error), "{error}"); +} + +/// The record a shard produces is sized by the fields around it, so a shard of +/// any length the budget admits is admitted at the decoder too: a jumbo-frame +/// producer is not stopped by a fixed shard cap. +#[test] +fn a_larger_datagram_budget_admits_larger_shards() { + let jumbo = MediaFecPolicy { max_wire_bytes: 8_959, ..STANDARD }; + let chunks = frame(2, 3, 4_000, 4_000); + let records = protect_video_frame(&chunks, &jumbo, provenance()).unwrap(); + for record in &records { + let wire = encode_media_wire_record(record, provenance()).unwrap(); + assert!(decode_media_wire_record(&wire).is_ok()); + } + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err()); +} + +// --------------------------------------------------------------------------- +// Framing: what a block may and may not contain +// --------------------------------------------------------------------------- + +#[test] +fn a_block_never_spans_frames() { + let mut chunks = frame(20, 4, 30, 30); + chunks[2].frame_id = 21; + let error = protect_video_frame(&chunks, &STANDARD, provenance()).unwrap_err(); + assert!(matches!(error, MediaFecError::Invalid(_)), "{error}"); + + let mut chunks = frame(20, 4, 30, 30); + chunks[3].deadline_ticks += 1; + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err(), "parity would misstate the deadline"); + + let mut chunks = frame(20, 4, 30, 30); + chunks[1].session_id = "session-2".to_string(); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err()); +} + +#[test] +fn a_frame_must_be_whole_and_shard_shaped() { + let mut chunks = frame(30, 4, 30, 30); + chunks.pop(); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err(), "a missing chunk"); + + let mut chunks = frame(30, 4, 30, 30); + chunks[3] = chunks[2].clone(); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err(), "a repeated chunk"); + + let mut chunks = frame(30, 4, 30, 30); + chunks[1].payload.truncate(29); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err(), "a short chunk that is not last"); + + let mut chunks = frame(30, 4, 30, 30); + chunks[3].payload.push(0); + assert!(protect_video_frame(&chunks, &STANDARD, provenance()).is_err(), "a last chunk longer than the shard"); + + assert!(protect_video_frame(&[], &STANDARD, provenance()).is_err()); +} + +#[test] +fn chunks_may_arrive_in_any_order() { + let original = frame(31, 6, 30, 12); + let mut shuffled = original.clone(); + shuffled.reverse(); + assert_eq!( + protect_video_frame(&original, &STANDARD, provenance()).unwrap(), + protect_video_frame(&shuffled, &STANDARD, provenance()).unwrap() + ); +} + +#[test] +fn an_audio_block_is_four_contiguous_equal_length_packets() { + assert!(protect_audio_block(&audio_block(1, 20)[..3], &STANDARD, provenance()).is_err(), "three packets"); + let mut packets = audio_block(1, 20); + packets[2].packet_id += 1; + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "a gap in packet ids"); + let mut packets = audio_block(1, 20); + packets[1].pts_ticks += 1; + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "a gap in presentation time"); + let mut packets = audio_block(1, 20); + packets[3].payload.push(0); + let error = protect_audio_block(&packets, &STANDARD, provenance()).unwrap_err(); + assert!(error.to_string().contains("payload length"), "unequal payload lengths: {error}"); + let mut packets = audio_block(1, 20); + packets[0].codec = "pcm".to_string(); + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "mixed codecs"); + let mut packets = audio_block(1, 20); + packets[2].session_id = "session-2".to_string(); + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "mixed sessions"); + let mut packets = audio_block(1, 20); + packets[2].stream_id = "other".to_string(); + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "mixed streams"); + let mut packets = audio_block(1, 20); + packets[1].timebase_den = 44_100; + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "mixed timebases"); + let mut packets = audio_block(1, 20); + packets[3].duration_ticks = 480; + assert!(protect_audio_block(&packets, &STANDARD, provenance()).is_err(), "mixed durations"); +} + +#[test] +fn parity_carries_its_frames_deadline_unchanged() { + let original = frame(40, 40, 50, 50); + let (_, parity) = split(protect_video_frame(&original, &STANDARD, provenance()).unwrap()); + assert!(!parity.is_empty()); + for shard in &parity { + assert_eq!(shard.deadline_ticks, original[0].deadline_ticks); + assert_eq!(shard.pts_ticks, original[0].pts_ticks); + assert_eq!(shard.frame_id, 40); + assert_eq!(shard.fec_scheme, MEDIA_FEC_SCHEME_RS_GF256_V1); + } +} + +#[test] +fn audio_parity_carries_the_latest_deadline_in_its_block_and_recovery_uses_it() { + let mut packets = audio_block(50, 30); + packets[1].deadline_ticks = 99_999; // the block's latest, not its last + let parity = protect_audio_block(&packets, &STANDARD, provenance()).unwrap(); + assert!(parity.iter().all(|shard| shard.deadline_ticks == 99_999)); + + let held = vec![packets[0].clone(), packets[2].clone(), packets[3].clone()]; + let recovered = recover_audio_block(&parity[..1], &held).unwrap(); + assert_eq!(recovered.len(), 1); + let packet = &recovered[0]; + assert_eq!(packet.packet_id, 51, "id from the block's arithmetic"); + assert_eq!(packet.pts_ticks, packets[1].pts_ticks, "pts from the block's arithmetic"); + assert_eq!(packet.deadline_ticks, 99_999, "deadline from the block"); + assert_eq!(packet.payload, packets[1].payload); + assert_eq!(packet.codec, "opus"); +} + +// --------------------------------------------------------------------------- +// Send order: a burst must land as one loss per block +// --------------------------------------------------------------------------- + +#[test] +fn send_order_is_a_permutation_of_the_frames_records() { + for n in [1_u16, 2, 15, 16, 17, 31, 32, 33, 47, 100] { + let original = frame(60, n, 20, 9); + let records = protect_video_frame(&original, &STANDARD, provenance()).unwrap(); + let sizes = STANDARD.video_block_sizes(n); + let expected_parity: usize = sizes.iter().map(|&k| usize::from(STANDARD.video_parity_shards(k))).sum(); + assert_eq!(records.len(), usize::from(n) + expected_parity, "n = {n}"); + let mut data_indexes: Vec = records + .iter() + .filter_map(|r| match r { + GameCultMediaWireRecord::Video(c) => Some(c.chunk_index), + _ => None, + }) + .collect(); + data_indexes.sort_unstable(); + assert_eq!(data_indexes, (0..n).collect::>(), "n = {n}"); + } +} + +#[test] +fn equal_blocks_send_one_record_per_block_before_any_block_repeats() { + for n in [32_u16, 48, 64, 24, 20, 18] { + let sizes = STANDARD.video_block_sizes(n); + let blocks = sizes.len(); + assert!(sizes.windows(2).all(|pair| pair[0] == pair[1]), "n = {n} must split equally"); + let records = protect_video_frame(&frame(70, n, 20, 20), &STANDARD, provenance()).unwrap(); + let order: Vec = records.iter().map(|r| block_of(r, &sizes)).collect(); + for window in order.windows(blocks) { + let mut seen = window.to_vec(); + seen.sort_unstable(); + seen.dedup(); + assert_eq!(seen.len(), blocks, "n = {n}: a window of {blocks} repeats a block: {order:?}"); + } + } +} + +/// Blocks that differ in size run out at the end and cannot keep the full +/// spacing. What holds: two records of one block are never closer than the +/// number of blocks still sending from the later one onward. +#[test] +fn unequal_blocks_keep_every_block_spaced_by_the_blocks_still_sending() { + for n in [17_u16, 19, 35, 40, 50, 100] { + let sizes = STANDARD.video_block_sizes(n); + assert!(sizes.windows(2).any(|pair| pair[0] != pair[1]), "n = {n} must split unequally"); + let records = protect_video_frame(&frame(71, n, 20, 20), &STANDARD, provenance()).unwrap(); + let order: Vec = records.iter().map(|r| block_of(r, &sizes)).collect(); + for later in 0..order.len() { + let Some(earlier) = (0..later).rev().find(|&i| order[i] == order[later]) else { continue }; + let mut still_sending = order[later..].to_vec(); + still_sending.sort_unstable(); + still_sending.dedup(); + assert!( + later - earlier >= still_sending.len(), + "n = {n}: block {} sent at {earlier} and again at {later} with {} blocks still sending: {order:?}", + order[later], + still_sending.len() + ); + } + } +} + +// --------------------------------------------------------------------------- +// The wire round trip, and the policy +// --------------------------------------------------------------------------- + +#[test] +fn a_frame_survives_the_wire_and_recovers_from_what_arrives() { + let original = frame(80, 20, 300, 111); + let sizes = STANDARD.video_block_sizes(20); + let records = protect_video_frame(&original, &STANDARD, provenance()).unwrap(); + + // The network eats a burst of four consecutive records. + let arrived: Vec = records + .iter() + .enumerate() + .filter(|(slot, _)| !(9..13).contains(slot)) + .map(|(_, record)| { + let wire = encode_media_wire_record(record, provenance()).unwrap(); + decode_media_wire_record(&wire).unwrap() + }) + .collect(); + + let mut all = Vec::new(); + for block in 0..sizes.len() { + let mut data = Vec::new(); + let mut parity = Vec::new(); + for record in arrived.iter().filter(|r| block_of(r, &sizes) == block) { + match record { + GameCultMediaWireRecord::Video(c) => data.push(c.clone()), + GameCultMediaWireRecord::VideoParity(p) => parity.push(p.clone()), + _ => unreachable!(), + } + } + let recovered = recover_video_block(&parity, &data).expect("a burst of 4 over 2 blocks is 2 per block"); + all.extend(data); + all.extend(recovered); + } + all.sort_by_key(|chunk| chunk.chunk_index); + assert_eq!(all, original); +} + +#[test] +fn a_custom_policy_sizes_blocks_by_its_own_fields() { + let policy = MediaFecPolicy { + video_max_block_data_shards: 8, + video_min_parity_shards: 1, + video_parity_divisor: 2, + ..STANDARD + }; + assert_eq!(policy.video_block_sizes(20), vec![7, 7, 6]); + assert_eq!(policy.video_parity_shards(7), 4); + assert_eq!(policy.video_parity_shards(1), 1); + let (data, parity) = split(protect_video_frame(&frame(90, 20, 20, 20), &policy, provenance()).unwrap()); + assert_eq!(data.len(), 20); + assert_eq!(parity.len(), 4 + 4 + 3); + assert_eq!(MediaFecPolicy::default(), STANDARD); +} + +#[test] +fn a_policy_that_cannot_form_a_block_is_refused() { + let zero = MediaFecPolicy { video_min_parity_shards: 0, ..STANDARD }; + assert!(protect_video_frame(&frame(1, 2, 10, 10), &zero, provenance()).is_err()); + let no_divisor = MediaFecPolicy { video_parity_divisor: 0, ..STANDARD }; + assert!(protect_video_frame(&frame(1, 2, 10, 10), &no_divisor, provenance()).is_err()); + let no_audio = MediaFecPolicy { audio_data_shards: 0, ..STANDARD }; + let error = protect_audio_block(&audio_block(1, 10), &no_audio, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "non-zero"), "{error}"); + let no_audio_parity = MediaFecPolicy { audio_parity_shards: 0, ..STANDARD }; + let error = protect_audio_block(&audio_block(1, 10), &no_audio_parity, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "non-zero"), "{error}"); + let too_wide = MediaFecPolicy { video_max_block_data_shards: 255, ..STANDARD }; + assert!(protect_video_frame(&frame(1, 2, 10, 10), &too_wide, provenance()).is_err()); + let audio_too_wide = MediaFecPolicy { audio_data_shards: 255, audio_parity_shards: 2, ..STANDARD }; + assert!(protect_audio_block(&audio_block(1, 10), &audio_too_wide, provenance()).is_err()); +} + +#[test] +fn parity_records_survive_the_positional_encoding_unchanged() { + let (_, parity) = split(protect_video_frame(&frame(5, 5, 40, 17), &STANDARD, provenance()).unwrap()); + let audio = protect_audio_block(&audio_block(5, 25), &STANDARD, provenance()).unwrap(); + for shard in parity { + let bytes = rmp_serde::to_vec(&shard).unwrap(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), shard); + } + for shard in audio { + let bytes = rmp_serde::to_vec(&shard).unwrap(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), shard); + } +} + +// --------------------------------------------------------------------------- +// Rulings pinned: policy bounds, block identity, audio arithmetic, bursts +// --------------------------------------------------------------------------- + +fn is_invalid_mentioning(error: &MediaFecError, needle: &str) -> bool { + matches!(error, MediaFecError::Invalid(message) if message.contains(needle)) +} + +fn audio_run(count: u64, payload_bytes: usize) -> Vec { + (0..count) + .map(|index| GameCultMediaAudioPacketRecord { + stream_id: "muninn.raven.av.rudp".to_string(), + session_id: "session-1".to_string(), + packet_id: 500 + index, + codec: "opus".to_string(), + pts_ticks: 960 * index as i64, + duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 960 * index as i64 + 5_000, + payload: bytes(index as u32, payload_bytes), + }) + .collect() +} + +/// A block is at most 256 shards, data and parity together, and a block of +/// exactly 256 is admitted. +#[test] +fn a_policy_admits_256_shards_per_block_and_refuses_257() { + let audio_ok = MediaFecPolicy { audio_data_shards: 250, audio_parity_shards: 6, ..STANDARD }; + let parity = protect_audio_block(&audio_run(250, 16), &audio_ok, provenance()).unwrap(); + assert_eq!(parity.len(), 6); + let audio_wide = MediaFecPolicy { audio_data_shards: 250, audio_parity_shards: 7, ..STANDARD }; + let error = protect_audio_block(&audio_run(250, 16), &audio_wide, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "256 shards"), "{error}"); + + // One video block of 250 chunks with 6 parity shards. + let video_ok = MediaFecPolicy { + video_max_block_data_shards: 250, + video_min_parity_shards: 6, + video_parity_divisor: 100, + ..STANDARD + }; + let records = protect_video_frame(&frame(1, 250, 4, 4), &video_ok, provenance()).unwrap(); + assert_eq!(records.len(), 256); + let video_wide = MediaFecPolicy { video_min_parity_shards: 7, ..video_ok }; + let error = protect_video_frame(&frame(1, 250, 4, 4), &video_wide, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "256 shards"), "{error}"); +} + +/// Two parity shards that differ only in where the block sits in the frame are +/// not one block, even when each is a well-formed record. +#[test] +fn parity_shards_that_differ_only_in_block_count_or_start_do_not_combine() { + let (_, parity) = split(protect_video_frame(&frame(7, 32, 40, 40), &STANDARD, provenance()).unwrap()); + let block_zero: Vec<_> = parity.iter().filter(|p| p.block_index == 0).cloned().collect(); + + let mut more_blocks = block_zero[1].clone(); + more_blocks.block_count += 1; + let error = recover_video_block(&[block_zero[0].clone(), more_blocks], &[]).unwrap_err(); + assert!(is_invalid_mentioning(&error, "one block"), "block_count: {error}"); + + let mut shifted = block_zero[1].clone(); + shifted.block_data_start += 1; + let error = recover_video_block(&[block_zero[0].clone(), shifted], &[]).unwrap_err(); + assert!(is_invalid_mentioning(&error, "one block"), "block_data_start: {error}"); +} + +/// The largest packet id and pts a block may end on are admitted and recover +/// to exactly those values; one tick or id more is refused everywhere: at the +/// wire, in recovery, and at the producer. +#[test] +fn an_audio_block_that_would_overflow_its_last_id_or_pts_is_refused() { + // Id: the block's last packet is u64::MAX. + let base = u64::MAX - 3; + let packets = audio_block(base, 20); + let parity = protect_audio_block(&packets, &STANDARD, provenance()).unwrap(); + let recovered = recover_audio_block(&parity, &packets[..3]).unwrap(); + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].packet_id, u64::MAX); + assert_eq!(recovered[0], packets[3]); + + let mut past_end = parity.clone(); + for shard in &mut past_end { + shard.base_packet_id += 1; + } + let error = recover_audio_block(&past_end, &[]).unwrap_err(); + assert!(is_invalid_mentioning(&error, "packet id or pts range"), "{error}"); + let wire = encode_media_wire_record( + &GameCultMediaWireRecord::AudioParity(past_end[0].clone()), + provenance(), + ) + .unwrap(); + assert!(decode_media_wire_record(&wire).unwrap_err().to_string().contains("packet id or pts range")); + let mut overflowing = audio_block(base, 20); + overflowing[0].packet_id += 1; + assert!(protect_audio_block(&overflowing, &STANDARD, provenance()).is_err()); + + // Pts: the block's last packet ends on i64::MAX. + let mut packets = audio_block(9, 20); + let first_pts = i64::MAX - 3 * 960; + for (index, packet) in packets.iter_mut().enumerate() { + packet.pts_ticks = first_pts + 960 * index as i64; + packet.deadline_ticks = i64::MAX; + } + let parity = protect_audio_block(&packets, &STANDARD, provenance()).unwrap(); + let recovered = recover_audio_block(&parity, &packets[..3]).unwrap(); + assert_eq!(recovered[0].pts_ticks, i64::MAX); + assert_eq!(recovered[0], packets[3]); + + let mut past_end = parity.clone(); + for shard in &mut past_end { + shard.base_pts_ticks += 1; + } + let error = recover_audio_block(&past_end, &[]).unwrap_err(); + assert!(is_invalid_mentioning(&error, "packet id or pts range"), "{error}"); + let mut overflowing = packets.clone(); + overflowing[1].pts_ticks += 1; + assert!(protect_audio_block(&overflowing, &STANDARD, provenance()).is_err()); +} + +/// Sixteen 8-byte chunks, the widest standard video block (m = 4). +/// `payload[i][j] = (i * 29 + j * 13 + 5) mod 256`. The expected bytes come +/// from an independent implementation of the construction in the module docs. +#[test] +fn known_answer_video_parity_bytes_for_k16_m4() { + let mut chunks = frame(2, 16, 8, 8); + for chunk in &mut chunks { + let i = usize::from(chunk.chunk_index); + chunk.payload = (0..8).map(|j| (i * 29 + j * 13 + 5) as u8).collect(); + } + let (_, parity) = split(protect_video_frame(&chunks, &STANDARD, provenance()).unwrap()); + let hexes: Vec = parity.iter().map(|shard| hex(&shard.payload)).collect(); + assert_eq!( + hexes, + ["5ed2f608c328adbe", "e46d011eabb600e2", "7c1168f2ba8a2343", "4f03d220d182f232"] + ); +} + +/// Blocks interleave, so a burst of consecutive losses costs each block a +/// share: the longest burst that leaves every frame whole, wherever it starts, +/// is the block count times the smallest parity count among the blocks. +#[test] +fn the_worst_burst_that_always_recovers_is_blocks_times_the_smallest_m() { + for chunk_count in 1..=399_u16 { + let sizes = STANDARD.video_block_sizes(chunk_count); + let parity_of: Vec = + sizes.iter().map(|&size| usize::from(STANDARD.video_parity_shards(size))).collect(); + let records = protect_video_frame(&frame(u64::from(chunk_count), chunk_count, 1, 1), &STANDARD, provenance()) + .unwrap(); + let blocks: Vec = records.iter().map(|record| block_of(record, &sizes)).collect(); + + // The shortest burst, over every start, that costs some block more + // than its m; the burst one shorter always recovers. + let mut shortest_failing = usize::MAX; + for start in 0..blocks.len() { + let mut lost = vec![0_usize; sizes.len()]; + for (offset, &block) in blocks[start..].iter().enumerate() { + lost[block] += 1; + if lost[block] > parity_of[block] { + shortest_failing = shortest_failing.min(offset + 1); + break; + } + } + } + let smallest_m = *parity_of.iter().min().unwrap(); + assert_eq!( + shortest_failing - 1, + sizes.len() * smallest_m, + "{chunk_count} chunks in {} blocks", + sizes.len() + ); + } +} + +// --------------------------------------------------------------------------- +// The ceiling, and what the audio budget check relies on +// --------------------------------------------------------------------------- + +/// Only the parity is measured against the budget, on the claim that it is the +/// largest record of an audio block. It carries every field a packet does plus +/// the stripe geometry, so this holds across the id, pts, duration and deadline +/// widths that vary the encoding. +#[test] +fn an_audio_parity_record_is_larger_on_the_wire_than_every_packet_it_protects() { + let bases: [u64; 8] = [0, 0x7c, 0xfc, 0xfffc, 0xffff_fffc, 99_997, u64::MAX - 3, 1]; + let starts: [i64; 7] = [i64::MIN, -1, 0, 125, 65_530, 4_294_967_290, i64::MAX - 3 * 4_000_000_000]; + let durations: [u32; 5] = [1, 120, 960, 65_535, 4_000_000_000]; + for base in bases { + for start in starts { + for duration in durations { + for wide_deadline in [false, true] { + let mut packets = audio_block(base, 40); + for (index, packet) in packets.iter_mut().enumerate() { + packet.pts_ticks = start + i64::from(duration) * index as i64; + packet.duration_ticks = duration; + packet.deadline_ticks = if wide_deadline { i64::MAX } else { packet.pts_ticks }; + } + let parity = protect_audio_block(&packets, &STANDARD, provenance()).unwrap(); + let parity_len = wire_len(&GameCultMediaWireRecord::AudioParity(parity[0].clone()), provenance()); + for packet in &packets { + let packet_len = wire_len(&GameCultMediaWireRecord::Audio(packet.clone()), provenance()); + assert!( + packet_len < parity_len, + "base {base} start {start} duration {duration}: packet {packet_len} >= parity {parity_len}" + ); + } + } + } + } + } +} + +/// A parity whose deadline precedes its last packet's pts would recover a packet +/// that fails `validate_audio_record`. Recovery refuses it, and so does the wire. +#[test] +fn audio_parity_with_a_deadline_before_its_last_packet_recovers_nothing() { + let packets = audio_block(10, 8); + let mut parity = protect_audio_block(&packets, &STANDARD, provenance()).unwrap(); + for shard in &mut parity { + shard.deadline_ticks = shard.base_pts_ticks; + } + let error = recover_audio_block(&parity, &packets[..2]).unwrap_err(); + assert!(is_invalid_mentioning(&error, "last pts_ticks"), "{error}"); + let wire = encode_media_wire_record( + &GameCultMediaWireRecord::AudioParity(parity[0].clone()), + provenance(), + ) + .unwrap(); + assert!(decode_media_wire_record(&wire).is_err()); +} + +/// The datagram budget is the producer's, but no budget may promise a record the +/// decoder would refuse. +#[test] +fn a_datagram_budget_over_the_media_record_ceiling_is_refused() { + let over = MediaFecPolicy { max_wire_bytes: GAMECULT_MEDIA_MAX_WIRE_BYTES + 1, ..STANDARD }; + let error = protect_audio_block(&audio_block(1, 100), &over, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "ceiling"), "{error}"); + let error = protect_video_frame(&frame(2, 3, 100, 100), &over, provenance()).unwrap_err(); + assert!(is_invalid_mentioning(&error, "ceiling"), "{error}"); +} + +/// A budget at the ceiling admits shards up to it, and the decoder accepts +/// what the policy admitted. +#[test] +fn a_budget_at_the_media_record_ceiling_admits_large_shards_end_to_end() { + let at_ceiling = MediaFecPolicy { max_wire_bytes: GAMECULT_MEDIA_MAX_WIRE_BYTES, ..STANDARD }; + let records = + protect_video_frame(&frame(2, 3, 60_000, 60_000), &at_ceiling, provenance()).unwrap(); + for record in &records { + let wire = encode_media_wire_record(record, provenance()).unwrap(); + assert!(wire.len() <= GAMECULT_MEDIA_MAX_WIRE_BYTES); + decode_media_wire_record(&wire).unwrap(); + } + let parity = protect_audio_block(&audio_block(1, 60_000), &at_ceiling, provenance()).unwrap(); + let wire = encode_media_wire_record( + &GameCultMediaWireRecord::AudioParity(parity[0].clone()), + provenance(), + ) + .unwrap(); + decode_media_wire_record(&wire).unwrap(); +} diff --git a/packages/cultnet-rs/tests/media_fec_loss.rs b/packages/cultnet-rs/tests/media_fec_loss.rs new file mode 100644 index 00000000..19b8fd20 --- /dev/null +++ b/packages/cultnet-rs/tests/media_fec_loss.rs @@ -0,0 +1,208 @@ +//! Video frames protected by the media FEC codec, sent through the impairment +//! schedules `cultnet_impair` applies to real sockets, with no sockets: each +//! record of a frame in send order takes the next datagram decision, and what +//! survives is handed to `recover_video_block`. +//! +//! Two kinds of claim. The guarantee is asserted: a block recovers exactly when +//! at most `m` of its records were lost, and a burst short enough that the send +//! order spreads it under `m` per block always leaves every frame whole. The +//! recovered-frame ratios are evidence, not assertions; `--nocapture` prints the +//! table and `tests/impairment/loss-matrix.txt` is that output (200 frames per cell, seed 20260930). + +#[allow(dead_code)] +#[path = "../examples/cultnet_impair/policy.rs"] +mod policy; + +use std::fmt::Write as _; +use std::path::Path; + +use cultnet_rs::{ + GameCultMediaVideoAccessUnitRecord, GameCultMediaWireRecord, MediaFecError, MediaFecPolicy, + MediaWireProvenance, protect_video_frame, recover_video_block, +}; +use policy::{Profile, Scheduler}; + +const POLICY: MediaFecPolicy = MediaFecPolicy::STANDARD; +const SEED: u64 = 20_260_930; +const FRAMES: u64 = 200; + +fn provenance() -> MediaWireProvenance<'static> { + MediaWireProvenance { + stored_at: "unix:1700000000000", + runtime_id: "raven-muninn-primary", + role: "muninn.media", + producer: "muninn", + } +} + +fn frame(frame_id: u64, chunk_count: u16) -> Vec { + (0..chunk_count) + .map(|index| GameCultMediaVideoAccessUnitRecord { + stream_id: "muninn.raven.av.rudp".to_string(), + session_id: "session-1".to_string(), + frame_id, + codec: "h264".to_string(), + pts_ticks: frame_id as i64 * 3_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + keyframe: false, + dependency_frame_id: frame_id.checked_sub(1), + deadline_ticks: frame_id as i64 * 3_000 + 22_500, + chunk_index: index, + chunk_count, + payload: (0..if index + 1 == chunk_count { 60 } else { 100 }) + .map(|byte| (frame_id as usize * 31 + usize::from(index) * 7 + byte) as u8) + .collect(), + }) + .collect() +} + +#[derive(Default)] +struct Tally { + frames: u64, + whole_without_parity: u64, + whole_with_parity: u64, + blocks_recovered: u64, + blocks_beyond_repair: u64, +} + +/// Runs one shape through one profile's schedule and asserts the per-block +/// guarantee on every block of every frame. +fn run(profile: &Profile, chunk_count: u16) -> Tally { + let mut scheduler = Scheduler::new(profile.clone(), SEED); + let sizes = POLICY.video_block_sizes(chunk_count); + let mut tally = Tally::default(); + + for frame_id in 0..FRAMES { + let original = frame(frame_id, chunk_count); + let records = protect_video_frame(&original, &POLICY, provenance()).unwrap(); + let survivors: Vec = records + .into_iter() + .filter(|_| !scheduler.decide(0).drop) + .collect(); + + let mut whole = true; + let mut whole_without_parity = true; + let mut start = 0_u16; + for (block, &size) in sizes.iter().enumerate() { + let m = usize::from(POLICY.video_parity_shards(size)); + let mut data = Vec::new(); + let mut parity = Vec::new(); + for record in &survivors { + match record { + GameCultMediaWireRecord::Video(chunk) + if chunk.chunk_index >= start && chunk.chunk_index < start + size => + { + data.push(chunk.clone()) + } + GameCultMediaWireRecord::VideoParity(shard) if usize::from(shard.block_index) == block => { + parity.push(shard.clone()) + } + _ => {} + } + } + let lost = usize::from(size) + m - data.len() - parity.len(); + let missing_data = usize::from(size) - data.len(); + whole_without_parity &= missing_data == 0; + + let block_whole = if missing_data == 0 { + true + } else if parity.is_empty() { + assert!(lost > m, "no parity survived, so more than m shards were lost"); + false + } else { + match recover_video_block(&parity, &data) { + Ok(recovered) => { + assert!(lost <= m, "block {block} of frame {frame_id} recovered {lost} losses with m = {m}"); + assert_eq!(recovered.len(), missing_data); + for chunk in &recovered { + assert_eq!( + chunk, &original[usize::from(chunk.chunk_index)], + "a recovered chunk must be the chunk that was sent" + ); + } + tally.blocks_recovered += 1; + true + } + Err(MediaFecError::BeyondRepair { .. }) => { + assert!(lost > m, "block {block} of frame {frame_id} refused a recoverable {lost} losses with m = {m}"); + tally.blocks_beyond_repair += 1; + false + } + Err(other) => panic!("{other}"), + } + }; + whole &= block_whole; + start += size; + } + tally.frames += 1; + tally.whole_with_parity += u64::from(whole); + tally.whole_without_parity += u64::from(whole_without_parity); + assert!(whole || !whole_without_parity, "parity never makes a whole frame worse"); + } + tally +} + +fn profile(name: &str) -> Profile { + Profile::load(&Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("tests/impairment/{name}.toml"))) + .unwrap() +} + +/// Every shape here splits into equal blocks, so a burst of `L` consecutive +/// datagrams costs each of `B` blocks at most `ceil(L / B)` of its records. +const EQUAL_BLOCK_SHAPES: [u16; 5] = [8, 16, 24, 32, 48]; + +#[test] +fn the_loss_matrix() { + let cells = ["clean", "loss-1pct", "loss-3pct", "loss-5pct", "burst-4", "burst-8"]; + let shapes = [8_u16, 16, 17, 24, 32, 48]; + let mut table = String::new(); + writeln!( + table, + "{:<10} {:>6} {:>7} {:>13} {:>12} {:>10} {:>9}", + "profile", "chunks", "frames", "whole/no-fec", "whole/fec", "recovered", "beyond" + ) + .unwrap(); + + for cell in cells { + let loaded = profile(cell); + for &chunks in &shapes { + let tally = run(&loaded, chunks); + writeln!( + table, + "{:<10} {:>6} {:>7} {:>13} {:>12} {:>10} {:>9}", + cell, + chunks, + tally.frames, + tally.whole_without_parity, + tally.whole_with_parity, + tally.blocks_recovered, + tally.blocks_beyond_repair + ) + .unwrap(); + + if cell == "clean" { + assert_eq!(tally.whole_with_parity, tally.frames, "a clean link loses nothing"); + } + let burst = match cell { + "burst-4" => 4_usize, + "burst-8" => 8, + _ => continue, + }; + if EQUAL_BLOCK_SHAPES.contains(&chunks) { + let sizes = POLICY.video_block_sizes(chunks); + let per_block = burst.div_ceil(sizes.len()); + let m = usize::from(POLICY.video_parity_shards(sizes[0])); + if per_block <= m { + assert_eq!( + tally.whole_with_parity, tally.frames, + "{cell} over {chunks} chunks costs each of {} blocks at most {per_block} of m = {m}: no frame may be lost", + sizes.len() + ); + } + } + } + } + println!("{table}"); +} diff --git a/packages/cultnet-rs/tests/media_stream_contracts.rs b/packages/cultnet-rs/tests/media_stream_contracts.rs index 963d78a2..08db4b30 100644 --- a/packages/cultnet-rs/tests/media_stream_contracts.rs +++ b/packages/cultnet-rs/tests/media_stream_contracts.rs @@ -5,9 +5,10 @@ use cultcache_rs::DatabaseEntry; use cultnet_rs::{ - GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, - GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, - GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_AUDIO_PARITY_SHARD_SCHEMA, + GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, + GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, GameCultMediaAudioPacketRecord, + GameCultMediaAudioParityShardRecord, GameCultMediaReceiverFeedbackRecord, GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, }; use rmpv::Value; @@ -129,9 +130,14 @@ fn parity_shards_carry_the_geometry_recovery_needs() { dependency_frame_id: Some(4241), deadline_ticks: 108_000, chunk_count: 5, + fec_scheme: "rs-gf256-v1".to_string(), + block_index: 0, + block_count: 1, + block_data_start: 0, + block_data_count: 5, parity_index: 1, parity_count: 2, - chunk_payload_bytes: 1024, + shard_payload_bytes: 1024, last_chunk_payload_bytes: 512, payload: vec![9, 9, 9], }; @@ -141,13 +147,21 @@ fn parity_shards_carry_the_geometry_recovery_needs() { panic!("parity shards must encode as a positional array"); }; - assert_eq!(slots.len(), 17, "field count is wire contract"); - assert_eq!(slots[12].as_u64(), Some(1), "slot 12 is parity_index"); - assert_eq!(slots[13].as_u64(), Some(2), "slot 13 is parity_count"); + assert_eq!(slots.len(), 22, "field count is wire contract"); + assert_eq!(slots[12].as_str(), Some("rs-gf256-v1"), "slot 12 is fec_scheme"); + assert_eq!(slots[13].as_u64(), Some(0), "slot 13 is block_index"); + assert_eq!(slots[16].as_u64(), Some(5), "slot 16 is block_data_count (k)"); + assert_eq!(slots[17].as_u64(), Some(1), "slot 17 is parity_index"); + assert_eq!(slots[18].as_u64(), Some(2), "slot 18 is parity_count (m)"); assert_eq!( - slots[15].as_u64(), + slots[20].as_u64(), Some(512), - "slot 15 is last_chunk_payload_bytes, which recovery needs to size the tail" + "slot 20 is last_chunk_payload_bytes, which recovery needs to size the tail" + ); + assert_eq!( + slots[21].as_slice(), + Some([9, 9, 9].as_slice()), + "the parity payload is bin, not an array of integers" ); let decoded: GameCultMediaVideoParityShardRecord = @@ -155,6 +169,42 @@ fn parity_shards_carry_the_geometry_recovery_needs() { assert_eq!(decoded, shard); } +#[test] +fn audio_parity_shards_carry_the_geometry_recovery_needs() { + let shard = GameCultMediaAudioParityShardRecord { + stream_id: "raven-primary-av".to_string(), + session_id: "session-1".to_string(), + codec: "opus".to_string(), + fec_scheme: "rs-gf256-v1".to_string(), + base_packet_id: 77, + base_pts_ticks: 960, + packet_duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 4_800, + data_shard_count: 4, + parity_index: 1, + parity_shard_count: 2, + shard_payload_bytes: 3, + payload: vec![7, 8, 9], + }; + + let encoded = rmp_serde::to_vec(&shard).expect("encodes"); + let Value::Array(slots) = decode_as_value(&encoded) else { + panic!("audio parity shards must encode as a positional array"); + }; + + assert_eq!(slots.len(), 15, "field count is wire contract"); + assert_eq!(slots[4].as_u64(), Some(77), "slot 4 is base_packet_id"); + assert_eq!(slots[10].as_u64(), Some(4), "slot 10 is data_shard_count (k)"); + assert_eq!(slots[12].as_u64(), Some(2), "slot 12 is parity_shard_count (m)"); + assert_eq!(slots[14].as_slice(), Some([7, 8, 9].as_slice()), "the payload is bin"); + + let decoded: GameCultMediaAudioParityShardRecord = + rmp_serde::from_slice(&encoded).expect("round-trips"); + assert_eq!(decoded, shard); +} + /// `missing_video_chunk_keys` was appended after producers were already writing /// feedback. A record written without it must still decode, or the append was a /// breaking change wearing a `default` attribute. @@ -205,6 +255,11 @@ fn published_schema_constants_match_the_derived_identity() { GameCultMediaAudioPacketRecord::SCHEMA_NAME, GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA ); + assert_eq!( + GameCultMediaAudioParityShardRecord::SCHEMA_NAME, + GAMECULT_MEDIA_AUDIO_PARITY_SHARD_SCHEMA + ); + assert_eq!(GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, "gamecult.media_video_parity_shard.v3"); assert_eq!( GameCultMediaReceiverFeedbackRecord::SCHEMA_NAME, GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA diff --git a/packages/cultnet-rs/tests/media_stream_wire.rs b/packages/cultnet-rs/tests/media_stream_wire.rs index cee31a04..9a3ae1be 100644 --- a/packages/cultnet-rs/tests/media_stream_wire.rs +++ b/packages/cultnet-rs/tests/media_stream_wire.rs @@ -2,7 +2,10 @@ //! nothing on the wire. use cultnet_rs::{ - GAMECULT_MEDIA_CHANNEL, GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + CultNetRudpDeliveredFrame, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, + GAMECULT_MEDIA_MAX_WIRE_BYTES, RudpTransportProfileOptions, create_rudp_transport_profile, + GAMECULT_MEDIA_CHANNEL, GameCultMediaAudioPacketRecord, GameCultMediaAudioParityShardRecord, + GameCultMediaReceiverFeedbackRecord, GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, GameCultMediaWireRecord, MediaWireProvenance, decode_media_wire_record, encode_media_wire_record, @@ -116,12 +119,34 @@ fn every_variant_round_trips() { dependency_frame_id: Some(0), deadline_ticks: 1, chunk_count: 4, + fec_scheme: "rs-gf256-v1".to_string(), + block_index: 0, + block_count: 1, + block_data_start: 0, + block_data_count: 4, parity_index: 1, parity_count: 2, - chunk_payload_bytes: 848, - last_chunk_payload_bytes: 400, + shard_payload_bytes: 3, + last_chunk_payload_bytes: 2, payload: vec![1, 2, 3], }), + GameCultMediaWireRecord::AudioParity(GameCultMediaAudioParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + codec: "opus".to_string(), + fec_scheme: "rs-gf256-v1".to_string(), + base_packet_id: 8, + base_pts_ticks: 0, + packet_duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 4_000, + data_shard_count: 4, + parity_index: 0, + parity_shard_count: 2, + shard_payload_bytes: 3, + payload: vec![4, 5, 6], + }), GameCultMediaWireRecord::Audio(GameCultMediaAudioPacketRecord { stream_id: "s".to_string(), session_id: "x".to_string(), @@ -215,3 +240,435 @@ fn provenance_must_be_complete() { fn the_media_channel_is_named_once() { assert_eq!(GAMECULT_MEDIA_CHANNEL, "media"); } + +fn video_parity() -> GameCultMediaVideoParityShardRecord { + GameCultMediaVideoParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + frame_id: 7, + codec: "h264".to_string(), + pts_ticks: 0, + duration_ticks: 1, + timebase_num: 1, + timebase_den: 90_000, + keyframe: false, + dependency_frame_id: None, + deadline_ticks: 1, + chunk_count: 20, + fec_scheme: "rs-gf256-v1".to_string(), + block_index: 1, + block_count: 2, + block_data_start: 10, + block_data_count: 10, + parity_index: 1, + parity_count: 3, + shard_payload_bytes: 4, + last_chunk_payload_bytes: 2, + payload: vec![1, 2, 3, 4], + } +} + +fn audio_parity() -> GameCultMediaAudioParityShardRecord { + GameCultMediaAudioParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + codec: "opus".to_string(), + fec_scheme: "rs-gf256-v1".to_string(), + base_packet_id: 8, + base_pts_ticks: 0, + packet_duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 4_000, + data_shard_count: 4, + parity_index: 1, + parity_shard_count: 2, + shard_payload_bytes: 3, + payload: vec![4, 5, 6], + } +} + +fn decodes(record: GameCultMediaWireRecord) -> Result { + let wire = encode_media_wire_record(&record, provenance()).map_err(|error| error.to_string())?; + decode_media_wire_record(&wire).map_err(|error| error.to_string()) +} + +/// A parity key names the block as well as the shard, or two blocks' parity +/// index 0 would share an address. +#[test] +fn parity_record_keys_name_the_block_and_the_shard() { + let mut other_block = video_parity(); + other_block.block_index = 0; + let mut other_shard = video_parity(); + other_shard.parity_index = 0; + let keys = [ + GameCultMediaWireRecord::VideoParity(video_parity()).record_key(), + GameCultMediaWireRecord::VideoParity(other_block).record_key(), + GameCultMediaWireRecord::VideoParity(other_shard).record_key(), + ]; + assert_eq!(keys[0], "s:x:video-parity:7:1:1"); + assert_ne!(keys[0], keys[1]); + assert_ne!(keys[0], keys[2]); + + let audio = GameCultMediaWireRecord::AudioParity(audio_parity()).record_key(); + assert_eq!(audio, "s:x:audio-parity:8:1"); + let mut other = audio_parity(); + other.parity_index = 0; + assert_ne!(audio, GameCultMediaWireRecord::AudioParity(other).record_key()); +} + +#[test] +fn a_well_formed_parity_record_is_admitted() { + assert!(decodes(GameCultMediaWireRecord::VideoParity(video_parity())).is_ok()); + assert!(decodes(GameCultMediaWireRecord::AudioParity(audio_parity())).is_ok()); +} + +/// The decoder is where a consumer's idea of a well-formed shard is enforced. +/// Each corruption below would otherwise reach the recovery arithmetic. +#[test] +fn a_malformed_video_parity_record_is_refused_with_its_reason() { + let cases: Vec<(&str, Box)> = vec![ + ("fec_scheme", Box::new(|r| r.fec_scheme = "xor-v0".to_string())), + ("block_index", Box::new(|r| r.block_index = 2)), + ("block_index", Box::new(|r| r.block_count = 0)), + ("inside the frame", Box::new(|r| r.block_data_start = 11)), + ("inside the frame", Box::new(|r| r.block_data_count = 0)), + ("parity_index", Box::new(|r| r.parity_index = 3)), + ("parity_index", Box::new(|r| r.parity_count = 0)), + ("256 shards", Box::new(|r| { + r.block_data_count = 254; + r.chunk_count = 300; + r.parity_count = 3; + })), + ("shard_payload_bytes", Box::new(|r| { + r.shard_payload_bytes = 0; + r.payload.clear(); + })), + ("last chunk length", Box::new(|r| r.last_chunk_payload_bytes = 0)), + ("last chunk length", Box::new(|r| r.last_chunk_payload_bytes = 5)), + ("declared shard length", Box::new(|r| r.payload.push(0))), + ("chunk_count", Box::new(|r| r.chunk_count = 0)), + ("256 shards", Box::new(|r| { + r.block_data_count = 256; + r.chunk_count = 300; + r.parity_count = 1; + r.parity_index = 0; + })), + ("deadline_ticks", Box::new(|r| r.deadline_ticks = -1)), + ]; + for (expected, corrupt) in cases { + let mut record = video_parity(); + corrupt(&mut record); + let error = decodes(GameCultMediaWireRecord::VideoParity(record)).unwrap_err(); + assert!(error.contains(expected), "expected {expected:?} in {error:?}"); + } +} + +#[test] +fn a_malformed_audio_parity_record_is_refused_with_its_reason() { + let cases: Vec<(&str, Box)> = vec![ + ("fec_scheme", Box::new(|r| r.fec_scheme = String::new())), + ("stripe", Box::new(|r| r.parity_index = 2)), + ("stripe", Box::new(|r| r.data_shard_count = 0)), + ("stripe", Box::new(|r| r.data_shard_count = 255)), + ("timing", Box::new(|r| r.packet_duration_ticks = 0)), + ("last pts_ticks", Box::new(|r| r.deadline_ticks = -1)), + ("packet id or pts range", Box::new(|r| r.base_packet_id = u64::MAX - 2)), + ("packet id or pts range", Box::new(|r| { + r.base_pts_ticks = i64::MAX - 960 * 3 + 1; + r.deadline_ticks = i64::MAX; + })), + ("declared shard length", Box::new(|r| { r.payload.pop(); })), + ("codec", Box::new(|r| r.codec = String::new())), + ("stream_id", Box::new(|r| r.stream_id = String::new())), + ("stream_id", Box::new(|r| r.session_id = String::new())), + ("timing", Box::new(|r| r.timebase_num = 0)), + ("timing", Box::new(|r| r.timebase_den = 0)), + ]; + for (expected, corrupt) in cases { + let mut record = audio_parity(); + corrupt(&mut record); + let error = decodes(GameCultMediaWireRecord::AudioParity(record)).unwrap_err(); + assert!(error.contains(expected), "expected {expected:?} in {error:?}"); + } +} + +/// The 256-shard ceiling and the deadline floor are inclusive on the side that +/// keeps a valid record valid. +#[test] +fn records_on_the_edge_of_each_bound_are_admitted() { + let mut video = video_parity(); + video.chunk_count = 300; + video.block_data_count = 254; + video.parity_count = 2; + video.parity_index = 1; + assert!(decodes(GameCultMediaWireRecord::VideoParity(video)).is_ok(), "254 + 2 = 256 shards"); + + let mut audio = audio_parity(); + audio.data_shard_count = 254; + audio.deadline_ticks = i64::MAX; + audio.parity_shard_count = 2; + assert!(decodes(GameCultMediaWireRecord::AudioParity(audio)).is_ok(), "254 + 2 = 256 shards"); + + let mut audio = audio_parity(); + audio.deadline_ticks = audio.base_pts_ticks + 3 * i64::from(audio.packet_duration_ticks); + assert!(decodes(GameCultMediaWireRecord::AudioParity(audio)).is_ok(), "a deadline at the last pts"); + + let mut audio = audio_parity(); + audio.base_packet_id = u64::MAX - 3; + assert!(decodes(GameCultMediaWireRecord::AudioParity(audio)).is_ok(), "the last packet id is u64::MAX"); + + let mut audio = audio_parity(); + audio.base_pts_ticks = i64::MAX - 960 * 3; + audio.deadline_ticks = i64::MAX; + assert!(decodes(GameCultMediaWireRecord::AudioParity(audio)).is_ok(), "the last pts is i64::MAX"); +} + +// --------------------------------------------------------------------------- +// The ceiling on one media record +// --------------------------------------------------------------------------- + +fn sized_audio_parity(payload_bytes: usize, deadline_ticks: i64) -> GameCultMediaAudioParityShardRecord { + GameCultMediaAudioParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + codec: "opus".to_string(), + fec_scheme: "rs-gf256-v1".to_string(), + base_packet_id: 8, + base_pts_ticks: 0, + packet_duration_ticks: 100, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks, + data_shard_count: 4, + parity_index: 0, + parity_shard_count: 2, + shard_payload_bytes: payload_bytes as u32, + payload: vec![7; payload_bytes], + } +} + +fn sized_video_parity(payload_bytes: usize) -> GameCultMediaVideoParityShardRecord { + GameCultMediaVideoParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + frame_id: 1, + codec: "h264".to_string(), + pts_ticks: 0, + duration_ticks: 1, + timebase_num: 1, + timebase_den: 90_000, + keyframe: false, + dependency_frame_id: Some(0), + deadline_ticks: 1, + chunk_count: 4, + fec_scheme: "rs-gf256-v1".to_string(), + block_index: 0, + block_count: 1, + block_data_start: 0, + block_data_count: 4, + parity_index: 1, + parity_count: 2, + shard_payload_bytes: payload_bytes as u32, + last_chunk_payload_bytes: 2, + payload: vec![7; payload_bytes], + } +} + +fn wire_len(record: &GameCultMediaWireRecord) -> usize { + encode_media_wire_record(record, provenance()).unwrap().len() +} + +/// The payload length whose wrapped record is exactly `wire_bytes` long, found +/// from the record's own linear growth in its payload. +fn payload_for_wire_len( + wire_bytes: usize, + wrap: &dyn Fn(usize) -> GameCultMediaWireRecord, +) -> usize { + let probe = 1_000; + let payload = probe + wire_bytes - wire_len(&wrap(probe)); + assert_eq!(wire_len(&wrap(payload)), wire_bytes, "the record grows one byte per payload byte here"); + payload +} + +/// RUDP would reassemble a 16 MiB record, and recovery work scales with the +/// shard length, so the decoder holds every record to one ceiling. +#[test] +fn a_media_record_one_byte_over_the_ceiling_is_refused_at_decode() { + let ceiling = GAMECULT_MEDIA_MAX_WIRE_BYTES; + let wraps: [(&str, Box GameCultMediaWireRecord>); 2] = [ + ( + "audio parity", + Box::new(|payload| GameCultMediaWireRecord::AudioParity(sized_audio_parity(payload, 300))), + ), + ( + "video parity", + Box::new(|payload| GameCultMediaWireRecord::VideoParity(sized_video_parity(payload))), + ), + ]; + for (name, wrap) in &wraps { + let at = payload_for_wire_len(ceiling, wrap.as_ref()); + let wire = encode_media_wire_record(&wrap(at), provenance()).unwrap(); + assert_eq!(wire.len(), ceiling); + decode_media_wire_record(&wire) + .unwrap_or_else(|error| panic!("{name} at the ceiling is refused: {error}")); + + // The encoder refuses to make an over-ceiling record, so one byte of + // padding stands in for a peer that sent one. + let mut over = wire.clone(); + over.push(0); + let error = decode_media_wire_record(&over).unwrap_err(); + assert!(error.to_string().contains("ceiling"), "{name}: {error}"); + } +} + +#[test] +fn a_jumbo_frame_sized_record_is_accepted() { + let wire = encode_media_wire_record( + &GameCultMediaWireRecord::AudioParity(sized_audio_parity(8_800, 300)), + provenance(), + ) + .unwrap(); + assert!(wire.len() > 8_800 && wire.len() < GAMECULT_MEDIA_MAX_WIRE_BYTES); + decode_media_wire_record(&wire).expect("a jumbo-frame record decodes"); +} + +/// Recovery hands every recovered packet the block's deadline, so a parity +/// whose deadline precedes the last packet's pts would recover a packet that +/// fails `validate_audio_record`. +#[test] +fn audio_parity_must_not_have_a_deadline_before_its_last_packets_pts() { + // Four packets, 100 ticks each from 0: the last plays at 300. + let decodes = |deadline_ticks: i64| { + let wire = encode_media_wire_record( + &GameCultMediaWireRecord::AudioParity(sized_audio_parity(8, deadline_ticks)), + provenance(), + ) + .unwrap(); + decode_media_wire_record(&wire) + }; + assert!(decodes(300).is_ok()); + assert!(decodes(301).is_ok()); + let error = decodes(299).unwrap_err(); + assert!(error.to_string().contains("last pts_ticks"), "{error}"); + let error = decodes(0).unwrap_err(); + assert!(error.to_string().contains("last pts_ticks"), "{error}"); +} + +// --------------------------------------------------------------------------- +// The media channel's own payload cap +// --------------------------------------------------------------------------- + +fn connected_session() -> CultNetRudpSession { + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 9, + ..CultNetRudpSessionOptions::default() + }); + session.assume_connected(0); + session +} + +const FRAGMENT_BYTES: usize = 16 * 1024; + +fn media_payload_limit(session_limit: Option) -> Option { + let profile = create_rudp_transport_profile( + "raven", + RudpTransportProfileOptions { + max_payload_bytes: session_limit, + ..RudpTransportProfileOptions::default() + }, + ); + profile.transports[0] + .channels + .iter() + .find(|channel| channel.channel_id == GAMECULT_MEDIA_CHANNEL) + .expect("the profile advertises the media channel") + .max_payload_bytes +} + +#[test] +fn the_media_channel_advertises_the_ceiling_or_a_tighter_session_cap() { + let ceiling = GAMECULT_MEDIA_MAX_WIRE_BYTES as u32; + assert_eq!(media_payload_limit(None), Some(ceiling)); + assert_eq!(media_payload_limit(Some(16 * 1024 * 1024)), Some(ceiling)); + assert_eq!(media_payload_limit(Some(4_096)), Some(4_096)); +} + +#[test] +fn a_session_refuses_to_send_more_than_the_ceiling_on_the_media_channel_only() { + let ceiling = GAMECULT_MEDIA_MAX_WIRE_BYTES; + let send = |channel: &str, payload: usize| { + connected_session().send_many( + channel, + vec![0; payload], + CultNetRudpSendOptions::default(), + Some(FRAGMENT_BYTES), + ) + }; + assert!(send(GAMECULT_MEDIA_CHANNEL, ceiling).is_ok()); + let error = send(GAMECULT_MEDIA_CHANNEL, ceiling + 1).unwrap_err(); + assert!(error.to_string().contains("max_payload_bytes"), "{error}"); + assert!(send("schema", ceiling + 1).is_ok(), "other channels keep the session cap"); +} + +/// Reassembly is where memory is spent, so the receiver enforces the ceiling +/// itself, not only the sender. +#[test] +fn a_session_refuses_to_reassemble_more_than_the_ceiling_on_the_media_channel() { + let ceiling = GAMECULT_MEDIA_MAX_WIRE_BYTES; + let packets = connected_session() + .send_many( + "schema", + vec![0; ceiling + 1], + CultNetRudpSendOptions::default(), + Some(FRAGMENT_BYTES), + ) + .unwrap(); + assert!(packets.len() > 1); + + let deliver = |channel: &str| -> anyhow::Result> { + let mut receiver = connected_session(); + let mut delivered = Vec::new(); + for packet in &packets { + let mut packet = packet.clone(); + packet.channel_id = channel.to_string(); + delivered.extend(receiver.receive(&packet, 0)?.delivered); + } + Ok(delivered) + }; + let frames = deliver("schema").expect("the same fragments reassemble on another channel"); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].payload.len(), ceiling + 1); + let error = deliver(GAMECULT_MEDIA_CHANNEL).unwrap_err(); + assert!(error.to_string().contains("max_payload_bytes"), "{error}"); +} + +/// A producer learns at encode time that a record is over the ceiling, rather +/// than the receiver dropping it. +#[test] +fn the_encoder_refuses_a_record_over_the_ceiling() { + let wrap = |payload: usize| GameCultMediaWireRecord::AudioParity(sized_audio_parity(payload, 300)); + let at = payload_for_wire_len(GAMECULT_MEDIA_MAX_WIRE_BYTES, &wrap); + assert!(encode_media_wire_record(&wrap(at), provenance()).is_ok()); + let error = encode_media_wire_record(&wrap(at + 1), provenance()).unwrap_err(); + assert!(error.to_string().contains("ceiling"), "{error}"); +} + +/// A record that arrives unfragmented is held to the same ceiling. +#[test] +fn a_session_refuses_an_unfragmented_media_packet_over_the_ceiling() { + let packets = connected_session() + .send_many( + "schema", + vec![0; GAMECULT_MEDIA_MAX_WIRE_BYTES + 1], + CultNetRudpSendOptions::default(), + None, + ) + .unwrap(); + assert_eq!(packets.len(), 1, "sent whole"); + let mut packet = packets[0].clone(); + assert!(connected_session().receive(&packet, 0).is_ok()); + packet.channel_id = GAMECULT_MEDIA_CHANNEL.to_string(); + let error = connected_session().receive(&packet, 0).unwrap_err(); + assert!(error.to_string().contains("max_payload_bytes"), "{error}"); +} diff --git a/packages/cultnet-rs/tests/rudp_connect_generation.rs b/packages/cultnet-rs/tests/rudp_connect_generation.rs new file mode 100644 index 00000000..b456162c --- /dev/null +++ b/packages/cultnet-rs/tests/rudp_connect_generation.rs @@ -0,0 +1,1001 @@ +//! A Connect's sequence says whether it repeats the one a session accepted, and +//! the handshake alone seeds the watermark that orders delivery. Each test puts +//! one rule in front of real sessions or real transports. + +use std::collections::BTreeSet; +use std::net::{SocketAddr, UdpSocket}; +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use cultnet_rs::*; + +const CONNECTION_ID: u32 = 0x4355_4c54; + +fn socket() -> Result { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(20)))?; + Ok(socket) +} + +fn session(initial_sequence: u32) -> CultNetRudpSession { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(initial_sequence), + resend_delay_ms: 250, + max_pending_reliable_packets: None, + }) +} + +fn send(from: &mut CultNetRudpSession, text: &str) -> Result { + from.send("schema", text.as_bytes().to_vec(), ordered()) +} + +fn ordered() -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + } +} + +fn names(frames: &[CultNetRudpDeliveredFrame]) -> Vec { + frames + .iter() + .map(|frame| String::from_utf8(frame.payload.clone()).unwrap()) + .collect() +} + +/// A completed handshake between two sessions; returns the Connect and Accept. +fn handshake( + client: &mut CultNetRudpSession, + server: &mut CultNetRudpSession, +) -> Result<(CultNetRudpPacket, CultNetRudpPacket)> { + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.receive(&accept, 0)?; + assert!(client.connected() && server.connected()); + Ok((connect, accept)) +} + +/// What the server owed the client when the session ended is gone. A client +/// that reconnects on the same session must still receive what the new server +/// session sends: the old session's lost write is not a hole in the new one. +#[test] +fn a_reconnect_on_the_same_session_is_not_stranded_by_a_write_the_ended_session_owed() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let _lost = server.send("schema", b"lost".to_vec(), ordered())?; + server.create_disconnect(b"gone".to_vec()); + + let connect = client.create_connect(1, Vec::new())?; + let accept = server.accept_connect(&connect, 1, Vec::new())?; + client.receive(&accept, 1)?; + let after = server.send("schema", b"after".to_vec(), ordered())?; + assert_eq!(names(&client.receive(&after, 2)?.delivered), ["after"]); + Ok(()) +} + +/// A server session that ended and then accepts a Connect from a new client +/// forgets what the old client sent: the new client's sequences are its own. +#[test] +fn a_server_that_accepts_a_new_client_forgets_the_old_clients_sequences() -> Result<()> { + let mut old_client = session(50); + let mut server = session(500); + handshake(&mut old_client, &mut server)?; + for name in ["c1", "c2", "c3"] { + let packet = old_client.send("schema", name.as_bytes().to_vec(), ordered())?; + assert_eq!(names(&server.receive(&packet, 1)?.delivered), [name]); + } + server.create_disconnect(b"restart".to_vec()); + + let mut new_client = session(2); + let connect = new_client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + new_client.receive(&accept, 2)?; + let frame = new_client.send("schema", b"fresh".to_vec(), ordered())?; + assert_eq!(names(&server.receive(&frame, 3)?.delivered), ["fresh"]); + Ok(()) +} + +/// The same Connect again is a retransmit: it does not reset the session, so +/// what the session already delivered stays delivered once. +#[test] +fn a_retransmitted_connect_after_data_has_flowed_does_not_reset_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (connect, _) = handshake(&mut client, &mut server)?; + let frame = client.send("schema", b"once".to_vec(), ordered())?; + assert_eq!(names(&server.receive(&frame, 1)?.delivered), ["once"]); + + assert!(server.connect_repeats(&connect)); + let reply = server.accept_connect(&connect, 2, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack, "the Accept was already acknowledged"); + assert_eq!(server.outstanding_reliable_packet_count(), 0, "a repeat queued something"); + assert!( + server.receive(&frame, 3)?.delivered.is_empty(), + "a retransmitted Connect made the server forget what it had delivered" + ); + Ok(()) +} + +/// A Connect that carries another sequence is not a repeat, even from a peer +/// that sent the same bytes. +#[test] +fn a_connect_with_another_sequence_is_not_a_repeat_whatever_it_carries() -> Result<()> { + let mut server = session(500); + let mut first = session(1); + let connect = first.create_connect(0, b"same payload".to_vec())?; + server.accept_connect(&connect, 0, Vec::new())?; + assert!(server.connect_repeats(&connect)); + + let mut restarted = session(9); + let again = restarted.create_connect(0, b"same payload".to_vec())?; + assert!(!server.connect_repeats(&again)); + let mut not_a_connect = connect.clone(); + not_a_connect.packet_type = CultNetRudpPacketType::Ping; + assert!(!server.connect_repeats(¬_a_connect)); + server.create_disconnect(Vec::new()); + assert!(!server.connect_repeats(&connect), "an ended session repeats nothing"); + Ok(()) +} + +/// An Accept counts only when its ack field names the pending Connect. +#[test] +fn an_accept_that_does_not_name_the_pending_connect_is_ignored() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + + let mut stale = accept.clone(); + stale.ack = connect.sequence + 50; + stale.ack_mask = 0; + client.receive(&stale, 1)?; + assert!(!client.connected(), "a stale Accept connected the session"); + assert_eq!(client.pending_reliable_sequences(), vec![connect.sequence]); + + let mut named_by_mask = accept.clone(); + named_by_mask.ack = connect.sequence + 3; + named_by_mask.ack_mask = 1 << 2; + client.receive(&named_by_mask, 2)?; + assert!(client.connected(), "the mask names the Connect"); + Ok(()) +} + +/// A late or duplicate Accept after a local disconnect must not revive the +/// ended session. +#[test] +fn an_accept_after_a_local_disconnect_does_not_reconnect_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (_, accept) = handshake(&mut client, &mut server)?; + client.create_disconnect(b"bye".to_vec()); + assert!(!client.connected()); + + client.receive(&accept, 5)?; + assert!(!client.connected(), "a late Accept revived an ended session"); + Ok(()) +} + +/// A client that reconnects to a restarted server is not silenced by what it +/// received from the old one: the new server's sequences may overlap the old +/// ones, and none of them is a duplicate. +#[test] +fn a_reconnect_forgets_what_the_old_server_sent() -> Result<()> { + let mut client = session(1); + let mut old_server = session(500); + handshake(&mut client, &mut old_server)?; + for name in ["d1", "d2", "d3"] { + let packet = send(&mut old_server, name)?; + assert_eq!(names(&client.receive(&packet, 1)?.delivered), [name]); + } + + let mut new_server = session(501); + let connect = client.create_connect(2, Vec::new())?; + let accept = new_server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + let fresh = send(&mut new_server, "fresh")?; + assert_eq!(names(&client.receive(&fresh, 3)?.delivered), ["fresh"]); + Ok(()) +} + +/// An Accept for a Connect that was abandoned before it was answered does not +/// connect the session. +#[test] +fn an_accept_for_an_abandoned_connect_does_not_connect_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.create_disconnect(b"never mind".to_vec()); + + client.receive(&accept, 1)?; + assert!(!client.connected(), "an Accept revived an abandoned Connect"); + Ok(()) +} + +/// What a session still owed a vanished peer does not keep the next peer out: +/// a new generation drops it before the queue is asked for room. +#[test] +fn a_full_queue_owed_to_a_vanished_peer_does_not_refuse_a_new_generation() -> Result<()> { + let bounded = |initial_sequence: u32| { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(initial_sequence), + resend_delay_ms: 250, + max_pending_reliable_packets: Some(1), + }) + }; + let mut server = bounded(500); + server.accept_connect(&session(1).create_connect(0, Vec::new())?, 0, Vec::new())?; + assert_eq!(server.outstanding_reliable_packet_count(), 1, "the Accept is owed and never acknowledged"); + let next = session(9).create_connect(1, Vec::new())?; + server.accept_connect(&next, 1, Vec::new())?; + assert!(server.connect_repeats(&next)); + + let mut client = bounded(1); + client.create_connect(0, Vec::new())?; + client.create_connect(1, Vec::new())?; + assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the new Connect is owed"); + Ok(()) +} + +/// An Accept counts once: a duplicate of the one honoured does not seed the +/// watermark again. +#[test] +fn a_duplicate_accept_does_not_reseed_the_watermark() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (_, accept) = handshake(&mut client, &mut server)?; + let s1 = server.send("schema", b"s1".to_vec(), ordered())?; + let s2 = server.send("schema", b"s2".to_vec(), ordered())?; + let mut forged = accept.clone(); + forged.sequence = s2.sequence; + client.receive(&forged, 1)?; + assert!(client.receive(&s2, 2)?.delivered.is_empty(), "s1 is still missing"); + assert_eq!(names(&client.receive(&s1, 3)?.delivered), ["s1", "s2"]); + Ok(()) +} + +/// Reliable data that arrives before the connecting side has the Accept has no +/// place in the order: it is not delivered and not acknowledged, so the sender +/// retransmits it after the handshake. +#[test] +fn reliable_data_before_the_accept_is_neither_delivered_nor_acknowledged() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + let a = server.send("schema", b"A".to_vec(), ordered())?; + let b = server.send("schema", b"B".to_vec(), ordered())?; + + for early in [&b, &a] { + assert!(client.receive(early, 1)?.delivered.is_empty()); + let ack = client.create_ack_for_received(early.sequence); + assert_eq!((ack.ack, ack.ack_mask), (0, 0), "an unhandled packet was acknowledged"); + } + client.receive(&accept, 2)?; + assert!(client.connected()); + assert_eq!(server.outstanding_reliable_packet_count(), 3, "nothing was acknowledged"); + + assert_eq!(names(&client.receive(&a, 3)?.delivered), ["A"]); + assert_eq!(names(&client.receive(&b, 3)?.delivered), ["B"]); + Ok(()) +} + +/// The other order: with the Accept in hand, frames that arrive out of order +/// are delivered in order. +#[test] +fn ordered_frames_after_the_accept_are_delivered_in_order_whatever_arrives_first() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let a = server.send("schema", b"A".to_vec(), ordered())?; + let b = server.send("schema", b"B".to_vec(), ordered())?; + assert!(client.receive(&b, 1)?.delivered.is_empty()); + assert_eq!(names(&client.receive(&a, 2)?.delivered), ["A", "B"]); + Ok(()) +} + +/// The options say nothing about the initial sequence; each session draws its +/// own, so one options object serves many sessions without giving them one +/// sequence. +#[test] +fn sessions_from_one_options_object_each_draw_their_own_initial_sequence() -> Result<()> { + fn assert_random(label: &str, draws: &BTreeSet) { + assert!(draws.len() > 32, "{label} does not draw at random: {draws:?}"); + assert!( + draws.iter().all(|value| (1..(1_u32 << 31)).contains(value)), + "{label} left [1, 2^31): {draws:?}" + ); + } + let address: SocketAddr = "127.0.0.1:9".parse()?; + assert_eq!(CultNetRudpSessionOptions::default().initial_sequence, None); + assert_eq!( + CultNetRudpSocketTransportOptions::client("c", socket()?, address, 1).initial_sequence, + None + ); + assert_eq!( + CultNetRudpSocketTransportOptions::server("s", socket()?, 1).initial_sequence, + None + ); + assert_eq!( + CultNetRudpServerHubOptions::new("h", socket()?, 1).initial_sequence, + None + ); + assert_eq!(CultMeshRudpSocketOptions::default().initial_sequence, None); + + let options = CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + ..Default::default() + }; + let connects: BTreeSet = (0..64) + .map(|_| Ok(CultNetRudpSession::new(options.clone()).create_connect(0, Vec::new())?.sequence)) + .collect::>()?; + assert_random("sessions from one options object", &connects); + Ok(()) +} + +/// A transport built from the same options each time (a reconnect loop's +/// factory) opens each connection with its own Connect sequence. +#[test] +fn a_reused_options_object_yields_a_different_connect_sequence_for_each_transport() -> Result<()> { + let listener = socket()?; + let endpoint = CultNetRudpEndpoint { host: "127.0.0.1".into(), port: listener.local_addr()?.port() }; + let options = CultMeshRudpSocketOptions::default(); + let mut sequences = BTreeSet::new(); + for _ in 0..8 { + let mut client = + CultMesh::create_rudp_client("client", CONNECTION_ID, &endpoint, options.clone())?; + client.connect(Vec::new())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = listener.recv_from(&mut buffer)?; + sequences.insert(decode_rudp_packet(&buffer[..received])?.sequence); + } + assert_eq!(sequences.len(), 8, "two connections opened with one Connect sequence: {sequences:?}"); + Ok(()) +} + +/// A hub builds a session for every client it admits; each draws its own +/// sequence, so no two clients are accepted with the same one. +#[test] +fn a_hub_accepts_each_client_with_its_own_sequence() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let mut accepts = BTreeSet::new(); + for _ in 0..8 { + let peer = socket()?; + let connect = session(1).create_connect(0, Vec::new())?; + peer.send_to(&encode_rudp_packet(&connect)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + accepts.insert(decode_rudp_packet(&buffer[..received])?.sequence); + } + assert_eq!(accepts.len(), 8, "two clients were accepted with one sequence: {accepts:?}"); + Ok(()) +} + +// Transports. + +fn hub_event_matching( + hub: &mut CultNetRudpServerHub, + matches: impl Fn(&CultNetRudpServerEvent) -> bool, +) -> Result { + for _ in 0..40 { + if let Some(event) = hub.receive_event_once()? + && matches(&event) + { + return Ok(event); + } + } + Err(anyhow!("the hub never produced the event")) +} + +fn client_on( + socket: UdpSocket, + remote: SocketAddr, + initial_sequence: u32, +) -> Result { + let mut options = CultNetRudpSocketTransportOptions::client("client", socket, remote, CONNECTION_ID); + options.initial_sequence = Some(initial_sequence); + CultNetRudpSocketTransportConnection::new(options) +} + +/// A client that restarts on the same address, connection id and Connect +/// payload is a new session, not a retransmit: the hub reports the old one +/// ended and delivers the new one's frames. +#[test] +fn the_hub_admits_a_restarted_client_with_the_same_address_id_and_payload() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let shared = socket()?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, hub_addr, 50_000)?; + first.connect(b"same".to_vec())?; + let CultNetRudpServerEvent::Connected { session: original } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))? + else { + unreachable!() + }; + first.receive_once()?; + for index in 0..3_u8 { + first.send("schema", vec![index])?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Frame { .. }))?; + } + + let mut second = client_on(twin, hub_addr, 7)?; + second.connect(b"same".to_vec())?; + let CultNetRudpServerEvent::Disconnected { session: ended, .. } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Disconnected { .. }))? + else { + unreachable!() + }; + assert_eq!(ended, original); + let CultNetRudpServerEvent::Connected { session: replacement } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))? + else { + unreachable!() + }; + assert_ne!(replacement.session_generation, original.session_generation); + second.receive_once()?; + second.send("schema", b"after restart".to_vec())?; + let CultNetRudpServerEvent::Frame { session, frame } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Frame { .. }))? + else { + unreachable!() + }; + assert_eq!(session, replacement); + assert_eq!(frame.payload, b"after restart"); + Ok(()) +} + +/// The same for a server-mode transport, where the restarted client's sequences +/// are all below what the old client used. +#[test] +fn server_mode_admits_a_restarted_client_and_delivers_its_frames() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let shared = socket()?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, server_addr, 50_000)?; + first.connect(b"same".to_vec())?; + assert!(server.receive_once()?.is_none()); + first.receive_once()?; + for index in 0..3_u8 { + first.send("schema", vec![index])?; + assert_eq!(server.receive_once()?.expect("first client frame").payload, vec![index]); + } + + let mut second = client_on(twin, server_addr, 7)?; + second.connect(b"same".to_vec())?; + assert!(server.receive_once()?.is_none()); + second.receive_once()?; + assert!(second.connected()); + second.send("schema", b"after restart".to_vec())?; + assert_eq!(server.receive_once()?.expect("restarted client frame").payload, b"after restart"); + Ok(()) +} + +/// A retransmitted Connect in server mode, after data has flowed, keeps the +/// session: the data it delivered is not delivered again. +#[test] +fn server_mode_keeps_its_session_when_a_connect_is_retransmitted_after_data_flowed() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let peer = socket()?; + let mut client = session(1); + let connect = client.create_connect(0, b"peer".to_vec())?; + peer.send_to(&encode_rudp_packet(&connect)?, server_addr)?; + server.receive_once()?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + client.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + let frame = client.send("schema", b"once".to_vec(), ordered())?; + peer.send_to(&encode_rudp_packet(&frame)?, server_addr)?; + assert_eq!(server.receive_once()?.expect("first delivery").payload, b"once"); + + peer.send_to(&encode_rudp_packet(&connect)?, server_addr)?; + server.receive_once()?; + peer.send_to(&encode_rudp_packet(&frame)?, server_addr)?; + assert!(server.receive_once()?.is_none(), "the retransmitted Connect reset the session"); + Ok(()) +} + +fn connected_client( + server_socket: &UdpSocket, + server_addr: SocketAddr, +) -> Result { + let mut client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::client( + "client", + socket()?, + server_addr, + CONNECTION_ID, + ))?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + // Another client's acknowledgements may be queued ahead of this Connect. + let (connect, client_addr) = loop { + let (received, from) = server_socket.recv_from(&mut buffer)?; + let packet = decode_rudp_packet(&buffer[..received])?; + if packet.packet_type == CultNetRudpPacketType::Connect { + break (packet, from); + } + }; + let accept = session(500).accept_connect(&connect, 0, Vec::new())?; + server_socket.send_to(&encode_rudp_packet(&accept)?, client_addr)?; + client.receive_once()?; + assert!(client.connected()); + Ok(client) +} + +/// A receipt belongs to the transport that issued it: asked of another +/// transport, it is not that transport's write, however alike their numbers. +#[test] +fn a_receipt_asked_of_another_transport_reads_invalidated() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut a = connected_client(&server, server_addr)?; + let mut b = connected_client(&server, server_addr)?; + let receipt_a = a.send_reliable("schema", b"a".to_vec())?; + let receipt_b = b.send_reliable("schema", b"b".to_vec())?; + assert_eq!(a.reliable_send_status(&receipt_a), CultNetRudpReliableSendStatus::Pending); + assert_eq!(b.reliable_send_status(&receipt_b), CultNetRudpReliableSendStatus::Pending); + assert_eq!(b.reliable_send_status(&receipt_a), CultNetRudpReliableSendStatus::Invalidated); + assert_eq!(a.reliable_send_status(&receipt_b), CultNetRudpReliableSendStatus::Invalidated); + assert_ne!(receipt_a, receipt_b, "receipts from two transports compare equal"); + Ok(()) +} + +/// A flush cut short by a timeout says so. +#[test] +fn a_flush_that_a_timeout_ended_names_the_timeout() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"unanswered".to_vec())?; + std::thread::sleep(Duration::from_millis(5)); + assert!(client.check_timeout(0)); + let error = client.flush_reliable(Duration::from_millis(50)).expect_err("the session timed out"); + assert!(error.to_string().contains("session timed out"), "{error}"); + Ok(()) +} + +// Hold-buffer boundaries: 1,024 frames and 64 channels are held; one more is +// refused. + +/// A connected pair with one sequence the receiver will never see, so every +/// later ordered frame waits behind it. The sender's window is kept open by +/// acknowledging what the receiver has. +fn held_behind_a_gap( + frames: usize, + channel_of: impl Fn(usize) -> String, +) -> Result> { + let mut sender = session(1); + let mut receiver = session(500); + handshake(&mut sender, &mut receiver)?; + let _missing = sender.send("gap", b"lost".to_vec(), ordered())?; + let mut held = 0; + for index in 0..frames { + let packet = sender.send(&channel_of(index), vec![0], ordered())?; + match receiver.receive(&packet, 1) { + Ok(result) => { + assert!(result.delivered.is_empty()); + held += 1; + } + Err(error) => return Ok(Err(error)), + } + if index % 16 == 15 { + sender.receive(&receiver.create_ack(), 1)?; + } + } + Ok(Ok(held)) +} + +#[test] +fn the_hold_buffer_takes_exactly_1024_frames_and_refuses_the_next() -> Result<()> { + assert_eq!(held_behind_a_gap(1024, |_| "schema".into())??, 1024); + let error = held_behind_a_gap(1025, |_| "schema".into())?.expect_err("1025 held frames"); + assert!(error.to_string().contains("hold buffer is full"), "{error}"); + Ok(()) +} + +#[test] +fn the_hold_buffer_takes_exactly_64_channels_and_refuses_the_next() -> Result<()> { + assert_eq!(held_behind_a_gap(64, |index| format!("channel-{index}"))??, 64); + let error = held_behind_a_gap(65, |index| format!("channel-{index}"))?.expect_err("65 held channels"); + assert!(error.to_string().contains("too many ordered channels"), "{error}"); + Ok(()) +} + +// Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed +// Connect from an earlier attempt is stale, a Connect from a new endpoint is a +// new client. + +/// Whether an acknowledgement names `sequence` by its ack field or its mask. +fn names_sequence(ack: &CultNetRudpPacket, sequence: u32) -> bool { + ack.ack == sequence + || (0..32).any(|bit| { + ack.ack_mask & (1_u32 << bit) != 0 && ack.ack > bit && ack.ack - bit - 1 == sequence + }) +} + +/// A frame the client's old generation already delivered, retransmitted after +/// the client reconnected, is acknowledged and not delivered into the new +/// generation. Every sequence the old generation issued is below the new +/// Connect. +#[test] +fn a_frame_from_the_clients_earlier_generation_is_acknowledged_not_delivered_again() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let old = send(&mut client, "old")?; + assert_eq!(names(&server.receive(&old, 1)?.delivered), ["old"]); + + let connect = client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + assert!(old.sequence < connect.sequence); + let late = server.receive(&old, 3)?; + assert!(late.delivered.is_empty(), "an earlier generation's frame was delivered again"); + let ack = server.create_ack_for_received(old.sequence); + assert!(names_sequence(&ack, old.sequence), "the duplicate was not acknowledged: {ack:?}"); + + let fresh = send(&mut client, "fresh")?; + assert_eq!(names(&server.receive(&fresh, 4)?.delivered), ["fresh"]); + Ok(()) +} + +/// The other direction: the server's frame delivered before the client +/// reconnected is not delivered to the client's new generation. +#[test] +fn a_frame_from_the_servers_earlier_generation_is_acknowledged_not_delivered_again() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let old = send(&mut server, "old")?; + assert_eq!(names(&client.receive(&old, 1)?.delivered), ["old"]); + + let connect = client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + let late = client.receive(&old, 3)?; + assert!(late.delivered.is_empty(), "an earlier generation's frame was delivered again"); + assert!(names_sequence(&client.create_ack_for_received(old.sequence), old.sequence)); + + let fresh = send(&mut server, "fresh")?; + assert_eq!(names(&client.receive(&fresh, 4)?.delivered), ["fresh"]); + Ok(()) +} + +/// A frame far below the receiver's highest sequence is a duplicate that is +/// acknowledged by name; without the acknowledgement its sender retransmits it +/// for ever. +#[test] +fn a_duplicate_below_the_receive_window_is_acknowledged_by_name() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let first = send(&mut client, "first")?; + assert_eq!(names(&server.receive(&first, 1)?.delivered), ["first"]); + for _ in 0..4_200 { + let packet = send(&mut client, "x")?; + server.receive(&packet, 1)?; + client.receive(&server.create_ack(), 1)?; + } + let again = server.receive(&first, 2)?; + assert!(again.delivered.is_empty()); + let ack = server.create_ack_for_received(first.sequence); + assert_eq!(ack.ack, first.sequence, "a duplicate below the window was not acknowledged by name"); + Ok(()) +} + +/// A Connect delayed from an earlier attempt of the client that is now +/// connected does not restart the session: the server answers with an Ack and +/// the client's ordered frames keep flowing. +#[test] +fn a_delayed_connect_from_an_earlier_attempt_does_not_strand_the_client() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let earlier = client.create_connect(0, Vec::new())?; + let retried = client.create_connect(300, Vec::new())?; + let accept = server.accept_connect(&retried, 301, Vec::new())?; + client.receive(&accept, 302)?; + server.receive(&client.create_ack_for_received(accept.sequence), 302)?; + assert!(client.connected()); + let a = send(&mut client, "a")?; + assert_eq!(names(&server.receive(&a, 303)?.delivered), ["a"]); + client.receive(&server.create_ack_for_received(a.sequence), 303)?; + + assert!(server.connect_repeats(&earlier), "the delayed Connect starts nothing"); + let reply = server.accept_connect(&earlier, 304, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + client.receive(&reply, 305)?; + assert!(client.connected()); + + let b = send(&mut client, "b")?; + assert_eq!(names(&server.receive(&b, 306)?.delivered), ["b"], "the delayed Connect reset the server"); + client.receive(&server.create_ack_for_received(b.sequence), 307)?; + assert!(!client.pending_reliable_sequences().contains(&b.sequence)); + Ok(()) +} + +/// A stale Connect is the peer's echo from the past, not evidence it is alive; +/// a retransmit of the accepted Connect is. +#[test] +fn a_repeated_connect_refreshes_liveness_and_a_stale_one_does_not() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let earlier = client.create_connect(0, Vec::new())?; + let current = client.create_connect(1, Vec::new())?; + server.accept_connect(¤t, 0, Vec::new())?; + + server.accept_connect(¤t, 900, Vec::new())?; + assert!(!server.check_timeout(1_000, 500), "a repeated Connect did not refresh liveness"); + + server.accept_connect(&earlier, 1_400, Vec::new())?; + assert!(server.check_timeout(1_600, 500), "a stale Connect refreshed liveness"); + Ok(()) +} + +/// The window is serial: a sequence just before the current Connect's is stale +/// across the wrap of the 32-bit space, one just after is a new client. +#[test] +fn stale_connects_are_recognised_by_serial_arithmetic() -> Result<()> { + let mut server = session(500); + let current = session(3).create_connect(0, Vec::new())?; + server.accept_connect(¤t, 0, Vec::new())?; + let before_the_wrap = session(u32::MAX - 1).create_connect(0, Vec::new())?; + assert!(server.connect_repeats(&before_the_wrap)); + let after = session(4).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&after)); + let beyond_the_window = session(3 + 4_096).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&beyond_the_window)); + let just_inside = session(3_u32.wrapping_sub(4_095)).create_connect(0, Vec::new())?; + assert!(server.connect_repeats(&just_inside)); + let just_outside = session(3_u32.wrapping_sub(4_096)).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&just_outside)); + Ok(()) +} + +/// A restarted client whose random first sequence lands just before the +/// server's current Connect is answered with an Ack, never an Accept. It does +/// not retransmit that Connect for ever: once the attempt times out it starts +/// a fresh one with a newly drawn sequence, and is admitted. +#[test] +fn a_restarted_client_whose_first_sequence_is_stale_connects_after_redrawing() -> Result<()> { + let mut server = session(500); + let mut old = session(1_000); + let connect = old.create_connect(0, Vec::new())?; + let old_accept = server.accept_connect(&connect, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + + let mut restarted = session(900); + let first = restarted.create_connect(0, b"join".to_vec())?; + // The old client has not acknowledged its Accept yet, so that is the reply: + // it names the old Connect, not this one. + let reply = server.accept_connect(&first, 1, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&reply, 1)?; + assert!(!restarted.connected(), "an Accept for another Connect connected the client"); + server.receive(&old.create_ack_for_received(old_accept.sequence), 1)?; + let reply = server.accept_connect(&first, 2, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + restarted.receive(&reply, 2)?; + assert!(!restarted.connected()); + + let retransmitted = restarted.due_resends(1_000); + assert_eq!(retransmitted.len(), 1); + assert_eq!(retransmitted[0].sequence, first.sequence, "the attempt is still young"); + + let fresh = restarted.due_resends(3_500); + assert_eq!(fresh.len(), 1); + assert_eq!(fresh[0].packet_type, CultNetRudpPacketType::Connect); + assert_ne!(fresh[0].sequence, first.sequence, "the same Connect was retransmitted for ever"); + assert_eq!(fresh[0].payload, b"join", "the fresh attempt lost the Connect payload"); + + let accept = server.accept_connect(&fresh[0], 3_500, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 3_500)?; + assert!(restarted.connected()); + let frame = send(&mut restarted, "hello")?; + assert_eq!(names(&server.receive(&frame, 3_501)?.delivered), ["hello"]); + Ok(()) +} + +/// A client whose Connect was answered is not given a new Connect by the +/// clock: the attempt timeout belongs to an unanswered Connect. +#[test] +fn an_answered_connect_is_never_replaced_by_the_attempt_timeout() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let (connect, _) = handshake(&mut client, &mut server)?; + let resent = client.due_resends(60_000); + assert!( + resent.iter().all(|packet| packet.packet_type != CultNetRudpPacketType::Connect + || packet.sequence == connect.sequence), + "a connected client started a new attempt" + ); + assert!(client.connected()); + Ok(()) +} + +/// A pinned client that restarts on a new port against a server-mode transport +/// is a new client: the Connect starts a new generation and the client is +/// admitted, though its sequence is the one the old client's Connect carried. +#[test] +fn server_mode_admits_a_pinned_client_that_restarts_on_a_new_port() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let mut first = client_on(socket()?, server_addr, 1)?; + first.connect(b"join".to_vec())?; + assert!(server.receive_once()?.is_none()); + first.receive_once()?; + first.send("schema", b"hello".to_vec())?; + assert_eq!(server.receive_once()?.expect("first client frame").payload, b"hello"); + drop(first); + + let mut second = client_on(socket()?, server_addr, 1)?; + second.connect(b"join".to_vec())?; + assert!(server.receive_once()?.is_none()); + second.receive_once()?; + assert!(second.connected(), "the restarted client on a new port was not admitted"); + second.send("schema", b"after".to_vec())?; + assert_eq!(server.receive_once()?.expect("restarted client frame").payload, b"after"); + Ok(()) +} + +/// An accepted peer that never speaks again times out: accepting a Connect is +/// the first thing heard from it. +#[test] +fn an_accepted_peer_that_goes_silent_times_out() -> Result<()> { + let mut server = session(500); + server.accept_connect(&session(10).create_connect(0, Vec::new())?, 0, Vec::new())?; + assert!(server.check_timeout(100_000, 1_000)); + Ok(()) +} + +// Ack Cut 1d, batch 2: the connect-attempt timeout. + +/// A server that keeps a session answers a client's repeated or stale Connect +/// with an Ack that may name the client's pending Connect. That Ack says no +/// session started: only an Accept the client honours retires its Connect, so +/// the attempt still times out and is replaced. +#[test] +fn an_ack_naming_the_pending_connect_does_not_retire_it() -> Result<()> { + let mut server = session(500); + let mut old = session(1); + let old_accept = server.accept_connect(&old.create_connect(0, Vec::new())?, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + server.receive(&old.create_ack_for_received(old_accept.sequence), 0)?; + + let mut restarted = session(1); + let first = restarted.create_connect(0, b"join".to_vec())?; + let reply = server.accept_connect(&first, 1, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + assert!(names_sequence(&reply, first.sequence), "the Ack must name the pending Connect for this test to bite"); + restarted.receive(&reply, 1)?; + assert!(!restarted.connected()); + assert_eq!(restarted.pending_reliable_sequences(), vec![first.sequence], "an Ack retired the Connect"); + + let fresh = restarted.due_resends(3_000); + assert_eq!(fresh.len(), 1, "the client waits for ever with nothing to resend"); + assert_eq!(fresh[0].payload, b"join"); + let accept = server.accept_connect(&fresh[0], 3_000, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 3_000)?; + assert!(restarted.connected()); + Ok(()) +} + +/// The fresh attempt is the abandoned sequence plus the receive window less +/// one, so a late copy of the abandoned Connect is stale for the server that +/// took the fresh one: answered with an Ack, no restart, no stranding. +#[test] +fn a_late_copy_of_an_abandoned_connect_is_stale_once_its_replacement_is_accepted() -> Result<()> { + let mut server = session(500); + let mut client = session(10); + let abandoned = client.create_connect(0, Vec::new())?; + let fresh = client.due_resends(3_000); + assert_eq!(fresh.len(), 1); + assert_eq!(fresh[0].sequence, abandoned.sequence + 4_095); + let accept = server.accept_connect(&fresh[0], 3_001, Vec::new())?; + client.receive(&accept, 3_002)?; + server.receive(&client.create_ack_for_received(accept.sequence), 3_002)?; + assert!(client.connected()); + let a = send(&mut client, "a")?; + assert_eq!(names(&server.receive(&a, 3_003)?.delivered), ["a"]); + client.receive(&server.create_ack_for_received(a.sequence), 3_003)?; + + assert!(server.connect_repeats(&abandoned), "the late copy would restart the server"); + let reply = server.accept_connect(&abandoned, 3_004, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + client.receive(&reply, 3_005)?; + assert!(client.connected()); + let b = send(&mut client, "b")?; + assert_eq!(names(&server.receive(&b, 3_006)?.delivered), ["b"]); + Ok(()) +} + +/// A server whose generation sits exactly where the fresh attempt lands takes +/// it for a repeat; the attempt after that leaves the window and is admitted. +#[test] +fn a_server_sitting_at_the_jump_is_left_by_the_next_attempt() -> Result<()> { + let mut server = session(500); + let mut old = session(5_095); + let old_accept = server.accept_connect(&old.create_connect(0, Vec::new())?, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + server.receive(&old.create_ack_for_received(old_accept.sequence), 0)?; + + let mut restarted = session(1_000); + let first = restarted.create_connect(0, b"join".to_vec())?; + restarted.receive(&server.accept_connect(&first, 1, Vec::new())?, 1)?; + assert!(!restarted.connected()); + + let second = restarted.due_resends(3_000).remove(0); + assert_eq!(second.sequence, 5_095, "the jump lands on the server's generation"); + let reply = server.accept_connect(&second, 3_001, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + restarted.receive(&reply, 3_001)?; + assert!(!restarted.connected()); + + let third = restarted.due_resends(6_000).remove(0); + assert_eq!(third.sequence, 5_095 + 4_095); + let accept = server.accept_connect(&third, 6_001, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 6_001)?; + assert!(restarted.connected()); + Ok(()) +} + +fn receive_packet(socket: &UdpSocket) -> Result { + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = socket.recv_from(&mut buffer)?; + decode_rudp_packet(&buffer[..received]) +} + +/// A pinned client that restarts on the same address is a repeat by sequence, +/// so the hub answers with an Ack; the client's attempt times out, and the +/// fresh Connect (another sequence) starts a new generation the hub admits. +#[test] +fn the_hub_admits_a_pinned_client_that_restarts_on_the_same_address() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let peer = socket()?; + let mut first = session(1); + peer.send_to(&encode_rudp_packet(&first.create_connect(0, b"join".to_vec())?)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + let accept = receive_packet(&peer)?; + first.receive(&accept, 0)?; + peer.send_to(&encode_rudp_packet(&first.create_ack_for_received(accept.sequence))?, hub_addr)?; + hub.receive_event_once()?; + + let mut restarted = session(1); + peer.send_to(&encode_rudp_packet(&restarted.create_connect(0, b"join".to_vec())?)?, hub_addr)?; + hub.receive_event_once()?; + restarted.receive(&receive_packet(&peer)?, 1)?; + assert!(!restarted.connected()); + + let fresh = restarted.due_resends(3_000).remove(0); + peer.send_to(&encode_rudp_packet(&fresh)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + restarted.receive(&receive_packet(&peer)?, 3_001)?; + assert!(restarted.connected(), "the restarted pinned client was never admitted"); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/rudp_server_hub.rs b/packages/cultnet-rs/tests/rudp_server_hub.rs index 80cfde3b..ab4ea71c 100644 --- a/packages/cultnet-rs/tests/rudp_server_hub.rs +++ b/packages/cultnet-rs/tests/rudp_server_hub.rs @@ -168,7 +168,7 @@ fn hub_keeps_independent_peer_sessions_and_exposes_connect_evidence() -> Result< } #[test] -fn hub_fences_replaced_generations_and_does_not_replace_connect_retransmits() -> Result<()> { +fn hub_fences_replaced_generations() -> Result<()> { let server_socket = socket()?; let server_addr = server_socket.local_addr()?; let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( @@ -189,12 +189,6 @@ fn hub_fences_replaced_generations_and_does_not_replace_connect_retransmits() -> })?; let original = connect(&mut hub, &mut provider, &original_evidence)?; - provider.connect(original_evidence.clone())?; - for _ in 0..3 { - assert!(hub.receive_event_once()?.is_none()); - } - assert_eq!(hub.session(original.remote_addr), Some(&original)); - provider.connect(replacement_evidence.clone())?; let mut disconnected = None; for _ in 0..20 { @@ -253,7 +247,7 @@ fn identical_connect_retransmit_reuses_pending_accept_after_loss() -> Result<()> let client_socket = socket()?; let mut client_session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -292,13 +286,13 @@ fn identical_connect_retransmit_reuses_pending_accept_after_loss() -> Result<()> fn exact_reliable_acks_clear_more_than_one_ack_window_of_fragments() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -352,7 +346,7 @@ fn exact_reliable_acks_clear_more_than_one_ack_window_of_fragments() -> Result<( fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -360,7 +354,7 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { receiver.set_max_pending_fragment_sets(1)?; let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -428,13 +422,13 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { fn replay_history_is_bounded_and_idle_hub_sessions_expire() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 33f02794..a64097de 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -30,9 +30,13 @@ fn socket() -> Result { } fn raw_session(connection_id: u32) -> CultNetRudpSession { + raw_session_from(connection_id, 1) +} + +fn raw_session_from(connection_id: u32, initial_sequence: u32) -> CultNetRudpSession { CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 1, + initial_sequence: Some(initial_sequence), resend_delay_ms: 250, max_pending_reliable_packets: None, }) @@ -442,7 +446,7 @@ fn server_mode_ends_the_session_when_the_session_refuses_a_packet() -> Result<() #[test] fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); - hub.initial_sequence = u32::MAX; + hub.initial_sequence = Some(u32::MAX); assert!(CultNetRudpServerHub::new(hub).is_err()); let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); hub.max_pending_reliable_packets = Some(0); @@ -455,13 +459,13 @@ fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { ] { let addr = "127.0.0.1:9".parse()?; let mut options = build(socket()?, addr); - options.initial_sequence = u32::MAX; + options.initial_sequence = Some(u32::MAX); assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); let mut options = build(socket()?, addr); options.max_pending_reliable_packets = Some(0); assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); let mut options = build(socket()?, addr); - options.initial_sequence = u32::MAX - 1; + options.initial_sequence = Some(u32::MAX - 1); options.max_pending_reliable_packets = Some(1); assert!(CultNetRudpSocketTransportConnection::new(options).is_ok()); } @@ -725,7 +729,7 @@ fn a_flush_fails_when_a_new_endpoint_replaces_the_peer_mid_wait() -> Result<()> send_to(&peer_a, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"a".to_vec())?)?; let _ = server.receive_once()?; server.send_reliable("schema", b"for A".to_vec())?; - send_to(&peer_b, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"b".to_vec())?)?; + send_to(&peer_b, server_addr, &raw_session_from(CONNECTION_ID, 2).create_connect(0, b"b".to_vec())?)?; let error = server.flush_reliable(Duration::from_millis(300)).expect_err("A's write was forgotten"); assert!( @@ -745,8 +749,224 @@ fn a_repeated_connect_acknowledges_what_it_carries() -> Result<()> { let accept = server.accept_connect(&connect, 0, Vec::new())?; let mut repeat = connect.clone(); repeat.ack = accept.sequence; - let reply = server.answer_repeated_connect(&repeat, 1)?; + let reply = server.accept_connect(&repeat, 1, Vec::new())?; assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); assert_eq!(server.outstanding_reliable_packet_count(), 0); Ok(()) } + +/// A connected client session with one ordered write the peer never received. +fn client_with_a_lost_write() -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut client = raw_session(CONNECTION_ID); + let mut server = raw_session(CONNECTION_ID); + let accept = server.accept_connect(&client.create_connect(0, Vec::new())?, 0, Vec::new())?; + client.receive(&accept, 0)?; + client.send( + "schema", + b"owed to the old session".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }, + )?; + // A fragmented write larger than the send window leaves part of it queued. + client.send_many( + "schema", + vec![7u8; 40 * 8], + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }, + Some(8), + )?; + assert_eq!(client.outstanding_reliable_packet_count(), 41); + assert!(client.queued_reliable_packet_count() > 0); + Ok((client, server)) +} + +/// A write belongs to the session it was issued in. Whichever way that session +/// ends, the write it still owed dies with it: it is not retransmitted into a +/// later session, where the new peer would deliver it. +#[test] +fn every_way_a_session_ends_drops_the_writes_it_owed() -> Result<()> { + type End = fn(&mut CultNetRudpSession, &mut CultNetRudpSession) -> Result<()>; + let endings: [(&str, End); 4] = [ + ("peer Disconnect", |client, server| { + client.receive(&server.create_disconnect(b"bye".to_vec()), 1)?; + Ok(()) + }), + ("local disconnect", |client, _| { + client.create_disconnect(b"bye".to_vec()); + Ok(()) + }), + ("timeout", |client, _| { + assert!(client.check_timeout(1_000, 10)); + Ok(()) + }), + ("refusal", |client, _| { + client.end_refused_session(); + Ok(()) + }), + ]; + for (name, end) in endings { + let (mut client, mut server) = client_with_a_lost_write()?; + end(&mut client, &mut server)?; + assert_eq!(client.outstanding_reliable_packet_count(), 0, "{name}: the write survived the end"); + + let mut next_server = raw_session(CONNECTION_ID); + let connect = client.create_connect(2_000, Vec::new())?; + next_server.accept_connect(&connect, 2_000, Vec::new())?; + for resend in client.due_resends(60_000) { + let result = next_server.receive(&resend, 60_000)?; + assert!( + result.delivered.is_empty(), + "{name}: the old session's write was delivered in the next session" + ); + } + } + Ok(()) +} + +fn connected_client(server_socket: &UdpSocket, server_addr: SocketAddr) -> Result<(CultNetRudpSocketTransportConnection, CultNetRudpSession, SocketAddr)> { + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server_socket.recv_from(&mut buffer)?; + let mut peer = raw_session(CONNECTION_ID); + let accept = peer.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(server_socket, client_addr, &accept)?; + let _ = client.receive_once()?; + Ok((client, peer, client_addr)) +} + +/// A local `disconnect()` ends the session like any other end: the receipt of a +/// write the peer never acknowledged is Invalidated, stays so after a +/// reconnect, and the write is not carried into the new session. +#[test] +fn a_receipt_invalidated_by_a_local_disconnect_stays_invalidated_after_reconnecting() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + let receipt = client.send_reliable("schema", b"written before the disconnect".to_vec())?; + assert_eq!(client.reliable_send_status(&receipt), CultNetRudpReliableSendStatus::Pending); + + client.disconnect(b"bye".to_vec())?; + assert_eq!(client.reliable_send_status(&receipt), CultNetRudpReliableSendStatus::Invalidated); + assert_eq!(client.outstanding_reliable_packet_count(), 0, "the write survived the disconnect"); + + client.connect(b"hello again".to_vec())?; + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated, + "reconnecting brought the write back" + ); + assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the new Connect is owed"); + Ok(()) +} + +/// A flush that starts after the session ended has no live session to wait on. +#[test] +fn a_flush_fails_after_a_local_disconnect_or_a_timeout() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed".to_vec())?; + client.disconnect(b"bye".to_vec())?; + let error = client.flush_reliable(Duration::from_millis(200)).expect_err("the write was dropped"); + assert!(error.to_string().contains("ended before its reliable writes were acknowledged"), "{error}"); + + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed".to_vec())?; + std::thread::sleep(Duration::from_millis(5)); + assert!(client.check_timeout(1)); + let error = client.flush_reliable(Duration::from_millis(200)).expect_err("the write was dropped"); + assert!(error.to_string().contains("ended before its reliable writes were acknowledged"), "{error}"); + Ok(()) +} + +/// A flush that starts after a reconnect waits on the new session: the end of +/// the old one does not fail it. +#[test] +fn a_flush_started_after_a_reconnect_waits_on_the_new_session() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, client_addr) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed to the old session".to_vec())?; + client.disconnect(b"bye".to_vec())?; + client.connect(b"hello again".to_vec())?; + let connect = drain(&server)? + .into_iter() + .rfind(|packet| packet.packet_type == CultNetRudpPacketType::Connect) + .expect("the reconnect reached the peer"); + let accept = raw_session(CONNECTION_ID).accept_connect(&connect, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + client.flush_reliable(Duration::from_millis(500))?; + Ok(()) +} + +/// A server-mode transport that accepts a new Connect after its peer left has a +/// live session again, and a flush started in it waits on that session. +#[test] +fn a_flush_started_after_the_server_accepts_a_new_connect_waits_on_the_new_session() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut first = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &first.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + send_to(&peer, server_addr, &first.create_disconnect(b"bye".to_vec()))?; + let _ = server.receive_once()?; + + let mut next = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &next.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + let accept = drain(&peer)? + .into_iter() + .rfind(|packet| packet.packet_type == CultNetRudpPacketType::Accept) + .expect("the new Connect was accepted"); + next.receive(&accept, 0)?; + send_to(&peer, server_addr, &next.create_ack_for_received(accept.sequence))?; + let _ = server.receive_once()?; + + server.flush_reliable(Duration::from_millis(500))?; + Ok(()) +} + +/// The session ended, the peer did not notice: what it sent before the end is +/// still remembered, held frames included, so its retransmits are duplicates. +#[test] +fn an_ending_does_not_forget_what_was_received_from_the_peer() -> Result<()> { + let mut client = raw_session(CONNECTION_ID); + let mut server = raw_session(CONNECTION_ID); + let accept = server.accept_connect(&client.create_connect(0, Vec::new())?, 0, Vec::new())?; + client.receive(&accept, 0)?; + let ordered = CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }; + let s1 = client.send("schema", b"s1".to_vec(), ordered.clone())?; + let s2 = client.send("schema", b"s2".to_vec(), ordered)?; + assert!(server.receive(&s2, 1)?.delivered.is_empty(), "s1 is missing, so s2 is held"); + + assert!(server.check_timeout(1_000, 10)); + let released = server.receive(&s1, 1_001)?.delivered; + assert_eq!( + released.iter().map(|frame| frame.payload.clone()).collect::>(), + [b"s1".to_vec(), b"s2".to_vec()], + "the ending forgot the held frame" + ); + assert!(server.receive(&s2, 1_002)?.delivered.is_empty(), "the ending forgot what was received"); + Ok(()) +} diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 8950f638..9309b05b 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -106,6 +106,7 @@ export { createRudpTransportProfile, decodeRudpPacket, encodeRudpPacket, + rudpClientBindHost, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetRudpReconnectLoop, CultNetRudpSession, diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 606cd418..fa116c07 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -12,6 +12,7 @@ import { CultNetRudpSocketTransportConnection, decodeRudpPacket, encodeRudpPacket, + rudpClientBindHost, type CultNetRudpPacket, } from "./rudp"; import { CultNetPeer } from "./peer"; @@ -112,8 +113,9 @@ export async function invokeCultNetOperation( options: CultNetOperationClientOptions, ): Promise { const target = parseRudpEndpoint(endpoint); - const socket = createSocket("udp4"); - await bindSocket(socket, 0, "127.0.0.1"); + const bindHost = await rudpClientBindHost(target.host); + const socket = createSocket(bindHost.includes(":") ? "udp6" : "udp4"); + await bindSocket(socket, 0, bindHost); const transport = new CultNetRudpSocketTransportConnection({ mode: "client", runtimeId: options.runtimeId, diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index ae7ea838..23852a3c 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -1,7 +1,22 @@ +import { randomInt } from "node:crypto"; import { EventEmitter } from "node:events"; import { type RemoteInfo, type Socket } from "node:dgram"; +import { lookup } from "node:dns/promises"; import type { CultNetTransportProfile } from "./contracts"; + +/** + * The address an RUDP client binds when its caller names none: loopback for a + * loopback endpoint, otherwise the unspecified address of the endpoint's family. + * A socket bound to loopback cannot send off the host on Windows. + */ +export async function rudpClientBindHost(endpointHost: string): Promise { + const { address, family } = await lookup(endpointHost.replace(/^\[(.*)\]$/, "$1")); + if (family === 6) { + return address === "::1" ? "::1" : "::"; + } + return address.startsWith("127.") ? "127.0.0.1" : "0.0.0.0"; +} import { CultNetReconnectController, createCultNetReconnectPolicy, @@ -18,6 +33,34 @@ const RUDP_FIXED_HEADER_BYTES = 36; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; +/// How long a client keeps retransmitting a Connect nobody answered before it +/// abandons that attempt for a fresh one. A Connect the server answers with an +/// Ack, never an Accept, is one the server judged stale (see `acceptConnect`); +/// retransmitting the same sequence would never change its mind. +const RUDP_CONNECT_ATTEMPT_MS = 3_000; + +/// A sequence drawn from a secure source in [1, 2^31). The Connect's sequence +/// is what tells a peer whether a Connect repeats one it already accepted or +/// starts a new session, so two sessions must not share one by default. +function drawSequence(): number { + return randomInt(1, 2 ** 31); +} + +/// Whether `sequence` is at or before `mark` in serial order, within the +/// receive window. The compare is modular, so it holds across the wrap of the +/// 32-bit space; a sequence further back than the window is the duplicate +/// test's below-window clause, and one ahead of the mark is never before it. +function atOrBefore(sequence: number, mark: number): boolean { + return ((mark - sequence) >>> 0) < RUDP_RECEIVED_SEQUENCE_WINDOW; +} + +function packetAcknowledges(packet: CultNetRudpPacket, sequence: number): boolean { + if (packet.ack === sequence) return true; + for (let bit = 0; bit < 32; bit += 1) { + if ((packet.ackMask & (1 << bit)) !== 0 && packet.ack - bit - 1 === sequence) return true; + } + return false; +} export type CultNetRudpPacketType = | "connect" @@ -72,11 +115,6 @@ type PendingReliablePacket = { lastSentAtMs: number; }; -type PendingOrderedFrame = { - frame: CultNetRudpDeliveredFrame; - nextSequence: number; -}; - type FragmentBuffer = { channelId: string; reliable: boolean; @@ -150,6 +188,24 @@ export class CultNetRudpSession { readonly #nextSequencedByChannel = new Map(); #nextFragmentId = 1; #connected = false; + /// Advances every time a generation ends. Everything issued in a generation + /// (writes, flushes) belongs to it and dies with it. + #generation = 0; + /// True from the end of a generation until the next Connect or Accept begins + /// one. A flush started in that interval has no live generation to wait on. + #ended = false; + /// The sequence of the Connect that started the current generation: sent by + /// this side, or accepted from the peer. A Connect repeats exactly when the + /// session is connected and the Connect carries this sequence. + #connectSequence: number | undefined; + /// A Connect this side sent is unanswered. Only then is an Accept honoured. + #awaitingAccept = false; + /// The payload of the Connect this side sent, kept so a fresh attempt can + /// always be built whatever became of the pending packet. + #connectPayload = new Uint8Array(); + /// When the unanswered Connect first went out; `dueResends` abandons it for a + /// fresh attempt once `RUDP_CONNECT_ATTEMPT_MS` has passed. + #connectStartedAtMs = 0; readonly #maxPendingReliablePackets: number | undefined; #lastReceivedAtMs: number | undefined; #highestReceivedSequence: number | undefined; @@ -157,8 +213,15 @@ export class CultNetRudpSession { readonly #latestSequencedByChannel = new Map(); readonly #pendingReliable = new Map(); readonly #queuedReliable: CultNetRudpPacket[] = []; - readonly #orderedNextSequenceByChannel = new Map(); - readonly #orderedBuffers = new Map>(); + /// Every reliable sequence up to and including this one has been received + /// since the peer state was last reset. Only the handshake seeds it: the + /// peer's Connect on the accepting side, the peer's Accept on the connecting + /// side. Reliable data that arrives before that is refused. + #receivedThrough: number | undefined; + /// Ordered frames received but not yet deliverable, keyed by first sequence. + /// A frame is held for exactly one reason: a reliable sequence below it has + /// not arrived. + readonly #orderedHeld = new Map(); readonly #fragmentBuffers = new Map(); /// Matches cultnet-rs's `max_pending_fragment_sets`. A fragment set is only /// removed on successful reassembly, so a set that loses one fragment is @@ -169,7 +232,7 @@ export class CultNetRudpSession { constructor(options: CultNetRudpSessionOptions) { this.connectionId = toUint32(options.connectionId, "connectionId"); - this.#nextSequence = toUint32(options.initialSequence ?? 1, "initialSequence"); + this.#nextSequence = toUint32(options.initialSequence ?? drawSequence(), "initialSequence"); if (this.#nextSequence === 0xffff_ffff) { throw new Error("RUDP initialSequence must leave room for a reliable packet."); } @@ -211,23 +274,53 @@ export class CultNetRudpSession { return this.#lastReceivedAtMs; } - resetPeerState(): void { + get generation(): number { + return this.#generation; + } + + get ended(): boolean { + return this.#ended; + } + + /// The one way a session generation ends. The session stops being connected + /// and what it still owed the peer dies with it: a write not yet acknowledged + /// is dropped, so no later session retransmits it or credits an ack to it. + /// What was learned from the peer is not touched: the peer may not know the + /// session ended, and forgetting what it sent would let its retransmits be + /// delivered twice. + #endSession(): void { this.#connected = false; + this.#ended = true; + this.#awaitingAccept = false; + this.#generation += 1; + this.#pendingReliable.clear(); + this.#queuedReliable.splice(0); + } + + /// Ends the current generation and forgets everything learned from the peer; + /// sequence numbers already issued stay issued. + resetPeerState(): void { + this.#endSession(); this.#lastReceivedAtMs = undefined; this.#highestReceivedSequence = undefined; this.#receivedSequences.clear(); this.#nextSequencedByChannel.clear(); this.#latestSequencedByChannel.clear(); - this.#pendingReliable.clear(); - this.#queuedReliable.splice(0); - this.#orderedNextSequenceByChannel.clear(); - this.#orderedBuffers.clear(); + this.#receivedThrough = undefined; + this.#orderedHeld.clear(); this.#fragmentBuffers.clear(); this.#fragmentSetsEvicted = 0; } createConnect(nowMs = 0, payload = new Uint8Array()): CultNetRudpPacket { + // A session that has had a peer starts a new generation: nothing it learned + // from that peer describes the one this Connect reaches, and what it still + // owed that peer no longer takes room in the queue. + if (this.#connectSequence !== undefined) { + this.resetPeerState(); + } this.#ensureReliableCapacity(1); + this.#ended = false; const packet = this.#createPacket({ packetType: "connect", channelId: "control", @@ -235,24 +328,68 @@ export class CultNetRudpSession { ordered: true, payload, }); + this.#connectSequence = packet.sequence; + this.#awaitingAccept = true; + this.#connectStartedAtMs = nowMs; + this.#connectPayload = new Uint8Array(payload); this.#trackReliable(packet, nowMs); return packet; } + /// Whether `packet` is a Connect this session's current generation already + /// owns: a retransmit of the Connect that started it (the session is + /// connected and the sequence is that Connect's), or a stale copy of an + /// earlier attempt by the same client (a sequence before it, within the + /// receive window). Servers that keep one session per peer ask this to tell a + /// Connect that starts a new session from one that does not; anything else + /// that reaches `acceptConnect` starts a new generation. + connectRepeats(packet: CultNetRudpPacket): boolean { + return packet.packetType === "connect" + && this.#connected + && this.#connectSequence !== undefined + && (this.#connectSequence === packet.sequence + || this.#connectIsStale(packet.sequence, this.#connectSequence)); + } + + /// A Connect that precedes the current generation's within the receive window + /// is the client's earlier attempt, delayed in the network: a client that + /// retried never sends a lower sequence again. Restarting on it would strand + /// the client, which honours only the Accept for its newest Connect. The rule + /// is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): keep the + /// connection and answer with an Ack. A restarted client whose random initial + /// sequence lands in this window is answered the same way and abandons the + /// attempt for a fresh draw (`RUDP_CONNECT_ATTEMPT_MS`). + #connectIsStale(sequence: number, current: number): boolean { + return sequence !== current && atOrBefore(sequence, current); + } + + /// Answers a Connect. A repeat of the accepted one queues nothing, so a + /// Connect storm cannot grow the reliable queue: the reply is the Accept + /// still awaiting acknowledgement, or an Ack once it was acknowledged. A + /// stale copy of an earlier attempt gets the same reply and changes nothing + /// else: it is not evidence the peer is alive. Any other Connect ends the + /// current generation, forgets the peer and accepts a new one. acceptConnect(packet: CultNetRudpPacket, nowMs = 0, payload = new Uint8Array()): CultNetRudpPacket { this.#requireConnection(packet); if (packet.packetType !== "connect") { throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); } - // A Connect from a peer this session already accepted repeats. - if (this.#connected) { - return this.answerRepeatedConnect(packet, nowMs); + if (this.connectRepeats(packet)) { + if (this.#connectSequence === packet.sequence) { + this.#applyAcknowledgements(packet); + this.#rememberReceived(packet.sequence); + this.#lastReceivedAtMs = nowMs; + } + return this.#pendingAcceptForResend(nowMs) ?? this.createAck(); } + this.resetPeerState(); this.#ensureReliableCapacity(1); - this.#rememberReceived(packet.sequence); + this.#seedReceived(packet.sequence); this.#lastReceivedAtMs = nowMs; + this.#connectSequence = packet.sequence; this.#connected = true; + this.#ended = false; const response = this.#createPacket({ packetType: "accept", channelId: "control", @@ -264,24 +401,12 @@ export class CultNetRudpSession { return response; } - /** - * Answers a Connect from a peer this session has already accepted. The packet's - * sequence is remembered and nothing is queued, so a Connect storm cannot grow - * the reliable queue. The reply is the Accept still awaiting acknowledgement, - * or an Ack once it was acknowledged. - */ - answerRepeatedConnect(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpPacket { - this.#requireConnection(packet); - if (packet.packetType !== "connect") { - throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); - } - this.#applyAcknowledgements(packet); - this.#rememberReceived(packet.sequence); - this.#lastReceivedAtMs = nowMs; + /// The Accept still awaiting acknowledgement, resent. + #pendingAcceptForResend(nowMs: number): CultNetRudpPacket | undefined { const pendingAccept = [...this.#pendingReliable.values()] .find(pending => pending.packet.packetType === "accept"); if (!pendingAccept) { - return this.createAck(); + return undefined; } pendingAccept.lastSentAtMs = nowMs; return { @@ -369,15 +494,28 @@ export class CultNetRudpSession { receive(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpReceiveResult { this.#requireConnection(packet); - this.#applyAcknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the + // Accept names it. A late or duplicate one, or one from an earlier + // generation, must not seed the watermark or revive an ended session. + const honoursAccept = packet.packetType === "accept" + && this.#awaitingAccept + && this.#connectSequence !== undefined + && packetAcknowledges(packet, this.#connectSequence); + if (packet.packetType === "accept" && !honoursAccept) { + return { delivered: [], readyToSend: [] }; + } + // While this side's Connect awaits its Accept, only the Accept it honours + // retires it: an Ack that names the Connect (a server's reply to a repeat or + // a stale copy) says the server did not start a session. + if (honoursAccept || !this.#awaitingAccept) { + this.#applyAcknowledgements(packet); + } const readyToSend = this.#promoteQueuedReliable(nowMs); this.#lastReceivedAtMs = nowMs; - const expectedSequenceIfUninitialized = this.#highestReceivedSequence === undefined - ? packet.sequence - : this.#highestReceivedSequence + 1; - if (packet.packetType === "accept") { - this.#rememberReceived(packet.sequence); + if (honoursAccept) { + this.#awaitingAccept = false; + this.#seedReceived(packet.sequence); this.#connected = true; return { delivered: [], readyToSend }; } @@ -404,7 +542,7 @@ export class CultNetRudpSession { } if (packet.packetType === "disconnect") { - this.#connected = false; + this.#endSession(); return { delivered: [], readyToSend, @@ -417,39 +555,41 @@ export class CultNetRudpSession { return { delivered: [], readyToSend }; } - const isDuplicate = (packet.reliable ?? false) - && (this.#receivedSequences.has(packet.sequence) - || (this.#highestReceivedSequence !== undefined - && packet.sequence < this.#highestReceivedSequence - && this.#highestReceivedSequence - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW)); + // Reliable data before the handshake has seeded the watermark has no place + // in the order: refuse it unremembered, so it is not acknowledged and the + // sender retransmits it once the handshake is done. + if ((packet.reliable ?? false) && this.#receivedThrough === undefined) { + return { delivered: [], readyToSend }; + } + + const isDuplicate = (packet.reliable ?? false) && this.#wasReceived(packet.sequence); if (packet.reliable ?? false) this.#rememberReceived(packet.sequence); if (isDuplicate) { return { delivered: [], readyToSend }; } + const delivered: CultNetRudpDeliveredFrame[] = []; const reassembled = this.#reassemble(packet); - if (!reassembled) { - return { delivered: [], readyToSend }; - } - - if (!reassembled.ordered && (packet.sequenced ?? false)) { - const newestSequence = reassembled.nextSequence - 1; - const latestSequence = this.#latestSequencedByChannel.get(reassembled.frame.channelId); - if (latestSequence !== undefined && newestSequence <= latestSequence) { - return { delivered: [], readyToSend }; + if (reassembled) { + if (reassembled.ordered) { + this.#orderedHeld.set(reassembled.frame.sequence, reassembled.frame); + } else if (!(packet.sequenced ?? false)) { + delivered.push(reassembled.frame); + } else { + const newestSequence = reassembled.nextSequence - 1; + const latestSequence = this.#latestSequencedByChannel.get(reassembled.frame.channelId); + if (latestSequence === undefined || newestSequence > latestSequence) { + this.#latestSequencedByChannel.set(reassembled.frame.channelId, newestSequence); + delivered.push(reassembled.frame); + } } - this.#latestSequencedByChannel.set(reassembled.frame.channelId, newestSequence); - return { delivered: [reassembled.frame], readyToSend }; } - if (!reassembled.ordered) { - return { delivered: [reassembled.frame], readyToSend }; - } - - return { - delivered: this.#deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized), - readyToSend, - }; + // Any reliable packet may have advanced the watermark, a fragment or an + // unordered frame as much as an ordered one, so the drain runs after all of + // them. + delivered.push(...this.#drainOrdered()); + return { delivered, readyToSend }; } createAck(): CultNetRudpPacket { @@ -490,7 +630,12 @@ export class CultNetRudpSession { } createAckForReceived(sequence: number): CultNetRudpPacket { + // What `receive` refused is not acknowledged by name (the ack carries only + // what was received), so its sender retransmits it. const receivedSequence = toUint32(sequence, "received sequence"); + if (!this.#wasReceived(receivedSequence)) { + return this.createAck(); + } const { ack } = this.#ackState(); return ack >= receivedSequence && ack - receivedSequence <= 32 ? this.createAck() @@ -506,7 +651,7 @@ export class CultNetRudpSession { } createDisconnect(reason = new Uint8Array()): CultNetRudpPacket { - this.#connected = false; + this.#endSession(); return this.#createPacket({ packetType: "disconnect", channelId: "control", @@ -521,11 +666,15 @@ export class CultNetRudpSession { if (nowMs - this.#lastReceivedAtMs <= timeoutMs) { return false; } - this.#connected = false; + this.#endSession(); return true; } dueResends(nowMs: number): CultNetRudpPacket[] { + const fresh = this.#abandonUnansweredConnect(nowMs); + if (fresh) { + return [fresh]; + } const due: CultNetRudpPacket[] = []; for (const pending of this.#pendingReliable.values()) { if (nowMs - pending.lastSentAtMs >= this.resendDelayMs) { @@ -536,6 +685,27 @@ export class CultNetRudpSession { return due.sort((left, right) => left.sequence - right.sequence); } + /// A Connect unanswered for `RUDP_CONNECT_ATTEMPT_MS` is replaced, not + /// retransmitted further, by a Connect with the same payload and the + /// abandoned sequence plus the receive window less one. Nothing was ever sent + /// in an unanswered generation, so no sequence issued so far is owed. The jump + /// keeps a late copy of the abandoned Connect inside the new one's stale + /// window, so a server that took the new Connect answers the copy with an Ack + /// instead of restarting; and it leaves the stale window of whatever + /// generation the server holds, unless that generation sits exactly at the + /// jump, and then the next attempt leaves it. The first Connect of a session + /// is the only one drawn at random. + #abandonUnansweredConnect(nowMs: number): CultNetRudpPacket | undefined { + if (!this.#awaitingAccept || nowMs - this.#connectStartedAtMs < RUDP_CONNECT_ATTEMPT_MS) { + return undefined; + } + if (this.#connectSequence === undefined) { + return undefined; + } + this.#nextSequence = Math.min(this.#connectSequence + RUDP_RECEIVED_SEQUENCE_WINDOW - 1, 0xffff_fffe); + return this.createConnect(nowMs, this.#connectPayload); + } + #createPacket(packet: { packetType: CultNetRudpPacketType; channelId: string; @@ -616,8 +786,34 @@ export class CultNetRudpSession { } } + /// The handshake's one act on the watermark: the peer's Connect or Accept is + /// the first sequence of the session, whatever else it has sent. + #seedReceived(sequence: number): void { + this.#receivedThrough = sequence; + this.#rememberReceived(sequence); + } + + /// The one duplicate test: true for a sequence in the window that was + /// received, for one below the window, and for one at or before the + /// watermark. The watermark starts at the handshake's seed, so a frame the + /// peer sent in an earlier generation (every sequence it issued is below the + /// Connect that began this one) is a duplicate of something already delivered + /// and is acknowledged, never delivered again. + #wasReceived(sequence: number): boolean { + return this.#receivedSequences.has(sequence) + || (this.#highestReceivedSequence !== undefined + && sequence < this.#highestReceivedSequence + && this.#highestReceivedSequence - sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW) + || (this.#receivedThrough !== undefined && atOrBefore(sequence, this.#receivedThrough)); + } + #rememberReceived(sequence: number): void { this.#receivedSequences.add(sequence); + if (this.#receivedThrough !== undefined) { + let through = this.#receivedThrough; + while (this.#receivedSequences.has(through + 1)) through += 1; + this.#receivedThrough = through; + } if (this.#highestReceivedSequence === undefined || sequence > this.#highestReceivedSequence) { this.#highestReceivedSequence = sequence; } @@ -734,68 +930,20 @@ export class CultNetRudpSession { }; } - #deliverOrdered( - frame: CultNetRudpDeliveredFrame, - nextSequence: number, - expectedSequenceIfUninitialized: number, - ): CultNetRudpDeliveredFrame[] { - let next = this.#orderedNextSequenceByChannel.get(frame.channelId); - if (next === undefined) { - next = Math.min(expectedSequenceIfUninitialized, frame.sequence); - this.#orderedNextSequenceByChannel.set(frame.channelId, next); - } - - while (frame.sequence > next && this.#receivedSequences.has(next) && !this.#orderedBuffers.get(frame.channelId)?.has(next)) { - next += 1; - this.#orderedNextSequenceByChannel.set(frame.channelId, next); - } - - if (frame.sequence < next) { - return []; - } - - if (frame.sequence > next) { - let buffer = this.#orderedBuffers.get(frame.channelId); - if (!buffer) { - buffer = new Map(); - this.#orderedBuffers.set(frame.channelId, buffer); - } - buffer.set(frame.sequence, { frame, nextSequence }); - return []; - } - - this.#orderedNextSequenceByChannel.set(frame.channelId, nextSequence); - return [ - frame, - ...this.#drainOrdered(frame.channelId), - ]; - } - - #drainOrdered(channelId: string): CultNetRudpDeliveredFrame[] { - const delivered: CultNetRudpDeliveredFrame[] = []; - const buffer = this.#orderedBuffers.get(channelId); - if (!buffer) { - return delivered; - } - - let next = this.#orderedNextSequenceByChannel.get(channelId); - while (next !== undefined && buffer.has(next)) { - const pending = buffer.get(next)!; - buffer.delete(next); - delivered.push(pending.frame); - next = pending.nextSequence; - this.#orderedNextSequenceByChannel.set(channelId, next); - this.#skipReceivedNonChannelSequences(channelId); - } - return delivered; - } - - #skipReceivedNonChannelSequences(channelId: string): void { - let next = this.#orderedNextSequenceByChannel.get(channelId); - while (next !== undefined && this.#receivedSequences.has(next) && !this.#orderedBuffers.get(channelId)?.has(next)) { - next += 1; - this.#orderedNextSequenceByChannel.set(channelId, next); - } + /// Delivers, in sequence order, every held ordered frame whose first sequence + /// is at most one past the watermark. Ordered delivery has one owner: the + /// watermark. A frame this call does not deliver waits for the sequence below + /// it, and nothing else releases it. + #drainOrdered(): CultNetRudpDeliveredFrame[] { + if (this.#receivedThrough === undefined) return []; + const releasable = [...this.#orderedHeld.keys()] + .filter((sequence) => sequence <= this.#receivedThrough! + 1) + .sort((left, right) => left - right); + return releasable.map((sequence) => { + const frame = this.#orderedHeld.get(sequence)!; + this.#orderedHeld.delete(sequence); + return frame; + }); } #allocateFragmentId(): number { @@ -824,12 +972,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; - // A flush belongs to the session it started in. Each session start advances - // the generation; each session end records the generation it ended and why. - // A reconnect from a disconnect listener starts a new generation but cannot - // un-end the old one. - #generation = 0; - #endedGeneration = -1; + // Why the last generation ended. A report only: the session owns whether it + // ended. #endedReason: Uint8Array | undefined; readonly #stats: CultNetTransportStats = { bytesReceived: 0, @@ -892,7 +1036,6 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement if (this.#mode !== "client") { throw new Error("Only a client RUDP socket transport can initiate connect."); } - this.#generation += 1; this.#sendPacket(this.#session.createConnect(Date.now(), payload)); } @@ -910,11 +1053,14 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement async flush(timeoutMs = 30_000): Promise { const deadline = Date.now() + Math.max(0, timeoutMs); - const generation = this.#generation; + // A flush belongs to the generation it started in: whatever ends that + // generation, and whatever begins after it (a reconnect from a disconnect + // listener included), the writes being waited on are gone. + const generation = this.#session.generation; for (;;) { // An ended session forgot its unacknowledged writes; reporting them - // flushed would be a lie, even if a new session has since begun. - if (this.#endedGeneration >= generation) { + // flushed would be a lie. + if (this.#session.ended || this.#session.generation !== generation) { throw new Error( `RUDP session ended before its reliable writes were acknowledged: ${Buffer.from(this.#endedReason ?? new Uint8Array()).toString("utf8")}`, ); @@ -934,11 +1080,6 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } } - #endGeneration(reason: Uint8Array): void { - this.#endedGeneration = this.#generation; - this.#endedReason = reason; - } - ping(payload = new Uint8Array()): void { this.#sendPacket(this.#session.createPing(payload)); } @@ -946,7 +1087,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement checkTimeout(timeoutMs: number, nowMs = Date.now()): boolean { const timedOut = this.#session.checkTimeout(nowMs, timeoutMs); if (timedOut) { - this.#endGeneration(Buffer.from("session timed out", "utf8")); + this.#endedReason = Buffer.from("session timed out", "utf8"); this.emit("timeout"); this.emit("close"); } @@ -973,7 +1114,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); - this.#endGeneration(reason); + this.#endedReason = reason; try { this.#sendPacket(this.#session.createDisconnect(reason)); } catch { @@ -1008,7 +1149,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } - let movedEndpoint = false; + // A Connect from another endpoint is a new client, whatever sequence it + // carries: the endpoint moves and the Connect starts a new generation. Only + // a Connect from the peer's own endpoint can be its repeat. + let connectFromNewEndpoint = false; if (!this.#remoteHost || this.#remotePort === undefined) { if (this.#mode === "server" && packet.packetType !== "connect") { this.#stats.packetsDropped += 1; @@ -1018,9 +1162,9 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#remotePort = remote.port; } else if (remote.address !== this.#remoteHost || remote.port !== this.#remotePort) { if (this.#mode === "server" && packet.packetType === "connect") { - movedEndpoint = true; this.#remoteHost = remote.address; this.#remotePort = remote.port; + connectFromNewEndpoint = true; } else { this.#stats.packetsDropped += 1; return; @@ -1028,22 +1172,19 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } if (this.#mode === "server" && packet.packetType === "connect") { - // Only a Connect from the endpoint already accepted repeats, and the - // session answers it with the Accept owed. A Connect from another - // endpoint replaces the peer: the old session's writes die with it. - const repeated = this.#session.connected && !movedEndpoint; - if (!repeated) { - // A timed-out session already ended its generation; any other - // predecessor ends here, connected or not. - if (this.#endedGeneration < this.#generation) { - this.#endGeneration(Buffer.from("replaced by a new Connect", "utf8")); - } + // The session decides whether this Connect repeats the one it accepted; a + // repeat is answered with the Accept owed, anything else replaces the + // peer and the old session's writes die with it. A timed-out session + // already ended its generation; any other predecessor ends here. + if ((connectFromNewEndpoint || !this.#session.connectRepeats(packet)) && !this.#session.ended) { + this.#endedReason = Buffer.from("replaced by a new Connect", "utf8"); + } + if (connectFromNewEndpoint) { this.#session.resetPeerState(); } let accept: CultNetRudpPacket; try { accept = this.#session.acceptConnect(packet, Date.now()); - if (!repeated) this.#generation += 1; } catch { this.#stats.packetsDropped += 1; return; @@ -1081,7 +1222,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } satisfies CultNetTransportFrame); } if (result.disconnected) { - this.#endGeneration(result.disconnectReason ?? new Uint8Array()); + this.#endedReason = result.disconnectReason ?? new Uint8Array(); this.emit("disconnect", { reason: result.disconnectReason ?? new Uint8Array() }); this.emit("close"); return; diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 7d1a110f..83583cfb 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -5,10 +5,10 @@ import { Duplex } from "node:stream"; import dgram, { type Socket } from "node:dgram"; import { rmSync, mkdtempSync } from "node:fs"; import { join } from "node:path"; -import { tmpdir } from "node:os"; +import { networkInterfaces, tmpdir } from "node:os"; import { z } from "zod"; -import { encode } from "@msgpack/msgpack"; +import { decode as decodeMsgpack, encode } from "@msgpack/msgpack"; import { CultCache, SchemaConflictError, @@ -49,6 +49,7 @@ import { ghostlightAgentStateGeneratedContract, parseCultNetMessage, invokeCultNetOperation, + rudpClientBindHost, startCultNetOperationServer, validateGhostlightAgentStateGenerated, validateGhostlightAgentState, @@ -168,6 +169,14 @@ function createDuplexPair(): { a: Duplex; b: Duplex } { return { a, b }; } +/// Whether the packet's ack field and mask cover the sequence. +function acknowledges(packet: CultNetRudpPacket, sequence: number): boolean { + if (packet.ack === sequence) return true; + return packet.ack > sequence + && packet.ack - sequence <= 32 + && (packet.ackMask & (1 << (packet.ack - sequence - 1))) !== 0; +} + async function bindUdpSocket(): Promise { const socket = dgram.createSocket("udp4"); await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); @@ -502,33 +511,32 @@ test("rudp session handshake acks reliable connect and accept packets", () => { test("rudp session computes ack masks and clears pending reliable packets", () => { const sender = new CultNetRudpSession({ connectionId: 7, initialSequence: 10, resendDelayMs: 100 }); const receiver = new CultNetRudpSession({ connectionId: 7, initialSequence: 200, resendDelayMs: 100 }); - sender.receive({ packetType: "accept", connectionId: 7, sequence: 1, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 7, sequence: 2, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true, nowMs: 0 }); const second = sender.send("schema", Buffer.from("second"), { reliable: true, ordered: true, nowMs: 0 }); const third = sender.send("schema", Buffer.from("third"), { reliable: true, ordered: true, nowMs: 0 }); - assert.deepEqual(sender.pendingReliableSequences, [10, 11, 12]); + assert.deepEqual(sender.pendingReliableSequences, [11, 12, 13]); receiver.receive(first); receiver.receive(third); const ackWithGap = receiver.createAck(); - assert.equal(ackWithGap.ack, 12); - assert.equal(ackWithGap.ackMask, 0b10 | (1 << 9)); + assert.equal(ackWithGap.ack, 13); + assert.equal(ackWithGap.ackMask, 0b110); sender.receive(ackWithGap); - assert.deepEqual(sender.pendingReliableSequences, [11]); + assert.deepEqual(sender.pendingReliableSequences, [12]); receiver.receive(second); const fullAck = receiver.createAck(); - assert.equal(fullAck.ack, 12); - assert.equal(fullAck.ackMask, 0b11 | (1 << 9)); + assert.equal(fullAck.ack, 13); + assert.equal(fullAck.ackMask, 0b111); sender.receive(fullAck); assert.deepEqual(sender.pendingReliableSequences, []); }); test("rudp session schedules reliable resends until acked", () => { const session = new CultNetRudpSession({ connectionId: 99, initialSequence: 1, resendDelayMs: 100 }); - session.receive({ packetType: "accept", connectionId: 99, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 99, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); const sent = session.send("schema", Buffer.from("payload"), { reliable: true, ordered: true, nowMs: 10 }); assert.deepEqual(session.dueResends(90), []); @@ -599,7 +607,7 @@ test("rudp unreliable sequenced delivery is scoped to its channel", () => { test("rudp rejects unreliable ordered delivery", () => { const session = new CultNetRudpSession({ connectionId: 196 }); - session.receive({ packetType: "accept", connectionId: 196, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 196, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); assert.throws( () => session.send("schema", Buffer.from("cannot order what will not retransmit"), { ordered: true }), /ordered delivery requires reliability/, @@ -608,7 +616,7 @@ test("rudp rejects unreliable ordered delivery", () => { test("rudp session bounds pending reliable packets before enqueue", () => { const session = new CultNetRudpSession({ connectionId: 102, initialSequence: 1, maxPendingReliablePackets: 2 }); - session.receive({ packetType: "accept", connectionId: 102, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 102, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); session.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); session.send("schema", Buffer.from("second"), { reliable: true, ordered: true }); @@ -616,10 +624,10 @@ test("rudp session bounds pending reliable packets before enqueue", () => { () => session.send("schema", Buffer.from("third"), { reliable: true, ordered: true }), /reliable send queue is full/, ); - assert.deepEqual(session.pendingReliableSequences, [1, 2]); + assert.deepEqual(session.pendingReliableSequences, [2, 3]); const fragmented = new CultNetRudpSession({ connectionId: 103, initialSequence: 1, maxPendingReliablePackets: 3 }); - fragmented.receive({ packetType: "accept", connectionId: 103, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + fragmented.receive(new CultNetRudpSession({ connectionId: 103, initialSequence: 900 }).acceptConnect(fragmented.createConnect(0), 0), 0); assert.throws( () => fragmented.sendMany("schema", Buffer.from("fragment-me"), { reliable: true, ordered: true, maxFragmentBytes: 3 }), /reliable send queue is full/, @@ -630,8 +638,7 @@ test("rudp session bounds pending reliable packets before enqueue", () => { test("rudp session suppresses duplicates and delivers reliable ordered payloads in sequence", () => { const sender = new CultNetRudpSession({ connectionId: 123, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 123, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 123, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 123, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0)); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); const second = sender.send("schema", Buffer.from("second"), { reliable: true, ordered: true }); @@ -649,8 +656,7 @@ test("rudp session suppresses duplicates and delivers reliable ordered payloads test("rudp session skips received control packets while ordering schema payloads", () => { const sender = new CultNetRudpSession({ connectionId: 124, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 124, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 124, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 124, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); const control = sender.createAck(); @@ -667,8 +673,7 @@ test("rudp session skips received control packets while ordering schema payloads test("rudp session fragments and reassembles reliable ordered payloads", () => { const sender = new CultNetRudpSession({ connectionId: 456, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 456, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 456, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 456, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const packets = sender.sendMany("schema", Buffer.from("fragment-me-please"), { reliable: true, @@ -1427,8 +1432,7 @@ test("rudp session advances large fragment sets through a bounded reliable windo const connectionId = 457; const sender = new CultNetRudpSession({ connectionId, initialSequence: 1, resendDelayMs: 25 }); const receiver = new CultNetRudpSession({ connectionId, initialSequence: 100, resendDelayMs: 25 }); - sender.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const fragmentCount = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; const payload = Uint8Array.from( @@ -1578,14 +1582,13 @@ test("RUDP operation service correlates typed payload envelopes", async () => { test("rudp sequence-neutral acknowledgements interoperate with ordered receivers", () => { const sender = new CultNetRudpSession({ connectionId: 125, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 125, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 125, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 125, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const ack = sender.createAck(); const request = sender.send("schema", Buffer.from("snapshot"), { reliable: true, ordered: true }); assert.equal(ack.sequence, 0); - assert.equal(request.sequence, 1); + assert.equal(request.sequence, 2); assert.deepEqual(receiver.receive(ack).delivered, []); assert.deepEqual(receiver.receive(request).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), ["snapshot"]); }); @@ -1926,7 +1929,7 @@ test("rudp receiver evicts stranded fragment sets instead of refusing new ones", // Heimdall fragmented envelopes (max_fragment_bytes = 2048), so on the live // auth path this map grew for every lost fragment and was never swept. const receiver = new CultNetRudpSession({ connectionId: 9, initialSequence: 1, resendDelayMs: 50 }); - receiver.receive({ packetType: "accept", connectionId: 9, sequence: 1, ack: 0, ackMask: 0, channelId: "control" }); + receiver.acceptConnect(new CultNetRudpSession({ connectionId: 9, initialSequence: 1 }).createConnect(0), 0); // 200 fragment sets that each lose their second fragment. for (let id = 1; id <= 200; id += 1) { @@ -2070,7 +2073,7 @@ test("rudp socket transport ends the session that refuses a reliable frame and n // neither in the goodbye nor after it retransmits. const goodbye = received.find((p) => p.packetType === "disconnect")!; assert.equal(peer.receive(goodbye, 1).disconnected, true); - assert.ok(peer.pendingReliableSequences.includes(poison.sequence), "the refused frame was acknowledged"); + assert.equal(acknowledges(goodbye, poison.sequence), false, "the goodbye acknowledged the refused frame"); const before = received.length; toServer(poison); await waitFor(() => server.stats.packetsDropped === 2, "the retransmit dropped"); @@ -2316,6 +2319,47 @@ test("operation service frees the slots of abandoned sessions once they idle out } }); +test("an RUDP client binds loopback only for a loopback endpoint", async () => { + assert.equal(await rudpClientBindHost("10.77.0.1"), "0.0.0.0"); + assert.equal(await rudpClientBindHost("2001:db8::1"), "::"); + assert.equal(await rudpClientBindHost("[2001:db8::1]"), "::"); + assert.equal(await rudpClientBindHost("127.0.0.1"), "127.0.0.1"); + assert.equal(await rudpClientBindHost("127.0.0.2"), "127.0.0.1"); + assert.equal(await rudpClientBindHost("::1"), "::1"); + assert.equal(await rudpClientBindHost("[::1]"), "::1"); +}); + +// A socket bound to loopback cannot send off the host on Windows. On Linux it +// reaches a local non-loopback address, but from 127.0.0.1, so the source +// address shows which one the client bound. +test("an operation client for a non-loopback endpoint does not send from loopback", async (t) => { + const address = Object.values(networkInterfaces()) + .flat() + .find((entry) => entry && entry.family === "IPv4" && !entry.internal)?.address; + if (!address) { + t.skip("no non-loopback IPv4 interface on this host"); + return; + } + const listener = dgram.createSocket("udp4"); + await new Promise((resolve) => listener.bind(0, "0.0.0.0", resolve)); + const source = new Promise((resolve) => listener.once("message", (_wire, remote) => resolve(remote.address))); + const request: CultNetOperationRequestMessage = { + schemaVersion: "cultnet.operation_request.v0", + messageId: "bind-host", + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + }; + try { + await assert.rejects(invokeCultNetOperation(`rudp://${address}:${udpPort(listener)}`, request, { runtimeId: "bind-test", timeoutMs: 300 })); + assert.equal(await source, address); + } finally { + listener.close(); + } +}); + test("server-mode transport replaces its peer when a Connect arrives from another endpoint", async () => { const serverSocket = await bindUdpSocket(); const socketA = await bindUdpSocket(); @@ -2447,7 +2491,7 @@ test("a repeated Connect acknowledges what it carries", () => { const server = new CultNetRudpSession({ connectionId }); const connect = client.createConnect(0); const accept = server.acceptConnect(connect, 0); - const reply = server.answerRepeatedConnect({ ...connect, ack: accept.sequence }, 1); + const reply = server.acceptConnect({ ...connect, ack: accept.sequence }, 1); assert.equal(reply.packetType, "ack"); assert.equal(server.outstandingReliablePacketCount, 0); }); @@ -2584,3 +2628,190 @@ test("operation service sweeps a session that ran a handler once it idles out", await server.close(); } }); + +test("a flush started after a reconnect waits on the new session", async () => { + const peerSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x1020305b; + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(peerSocket), + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + let clientPort = 0; + peerSocket.on("message", (wire, remote) => { + clientPort = remote.port; + received.push(decodeRudpPacket(wire)); + }); + try { + client.connect(); + await waitFor(() => received.some((p) => p.packetType === "connect"), "the Connect"); + const first = new CultNetRudpSession({ connectionId }); + peerSocket.send(encodeRudpPacket(first.acceptConnect(received[0]!, 0)), clientPort, "127.0.0.1"); + await waitFor(() => client.connected, "the client connected"); + client.send("schema", Buffer.from("owed to the old session")); + client.checkTimeout(50, Date.now() + 10_000); + assert.equal(client.connected, false); + + client.connect(); + await waitFor(() => received.filter((p) => p.packetType === "connect").length === 2, "the reconnect"); + const second = new CultNetRudpSession({ connectionId }); + const reconnect = received.filter((p) => p.packetType === "connect")[1]!; + peerSocket.send(encodeRudpPacket(second.acceptConnect(reconnect, 0)), clientPort, "127.0.0.1"); + await waitFor(() => client.connected, "the client reconnected"); + await client.flush(500); + } finally { + peerSocket.close(); + client.close(); + } +}); + +test("a flush started after the server accepts a new Connect waits on the new session", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x1020305c; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + try { + const first = new CultNetRudpSession({ connectionId }); + toServer(first.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the first Accept"); + toServer(first.createDisconnect(Buffer.from("bye"))); + await waitFor(() => !server.connected, "the first session ended"); + + const next = new CultNetRudpSession({ connectionId }); + toServer(next.createConnect(0)); + await waitFor(() => received.filter((p) => p.packetType === "accept").length === 2, "the second Accept"); + const accept = received.filter((p) => p.packetType === "accept")[1]!; + next.receive(accept, 0); + toServer(next.createAckForReceived(accept.sequence)); + await waitFor(() => server.outstandingReliablePacketCount === 0, "the second Accept acknowledged"); + + await server.flush(500); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("server-mode transport admits a restarted client and keeps its session when a Connect is retransmitted", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x10203059; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + let received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + const reliable = { reliable: true, ordered: true }; + try { + const first = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = first.createConnect(0, Buffer.from("same")); + toServer(connect); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the first Accept"); + first.receive(received.find((p) => p.packetType === "accept")!, 0); + const sent = [0, 1, 2].map((index) => first.send("schema", Buffer.from(`c${index}`), reliable)); + sent.forEach(toServer); + await waitFor(() => frames.length === 3, "the first client's frames"); + + // A retransmitted Connect is a repeat: what was delivered stays delivered once. + toServer(connect); + toServer(sent[0]!); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual(frames, ["c0", "c1", "c2"]); + + // The same client restarted on the same address, connection id and payload. + received = []; + const second = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(second.createConnect(0, Buffer.from("same"))); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the restarted client's Accept"); + second.receive(received.find((p) => p.packetType === "accept")!, 0); + toServer(second.send("schema", Buffer.from("after restart"), reliable)); + await waitFor(() => frames.includes("after restart"), "the restarted client's frame"); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("RUDP operation service admits a restarted client on the same address", async () => { + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: request => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }), + }); + const connectionId = 0x43554c54; + const port = Number(new URL(server.endpoint).port); + const socket = await bindUdpSocket(); + let session = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const answered: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + const result = session.receive(packet, Date.now()); + if (packet.packetType === "accept" || packet.reliable) { + socket.send(encodeRudpPacket(session.createAckForReceived(packet.sequence)), port, "127.0.0.1"); + } + for (const frame of result.delivered) { + answered.push((decodeMsgpack(frame.payload) as { messageId: string }).messageId); + } + }); + const toServer = (packet: CultNetRudpPacket) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1"); + const request = (messageId: string) => Buffer.from(encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + } satisfies CultNetOperationRequestMessage))); + try { + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the first Accept"); + toServer(session.send("schema", request("before-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("before-restart"), "the first answer"); + + session = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the restarted client's Accept"); + toServer(session.send("schema", request("after-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("after-restart"), "the restarted client's answer"); + } finally { + socket.close(); + await server.close(); + } +}); diff --git a/packages/cultnet-ts/test/rudp-connect-generation.test.ts b/packages/cultnet-ts/test/rudp-connect-generation.test.ts new file mode 100644 index 00000000..cea61a6d --- /dev/null +++ b/packages/cultnet-ts/test/rudp-connect-generation.test.ts @@ -0,0 +1,535 @@ +// A Connect's sequence says whether it repeats the one a session accepted, and +// the handshake alone seeds the watermark that orders delivery. +import assert from "node:assert/strict"; +import test from "node:test"; +import { createSocket, type Socket } from "node:dgram"; +import { once } from "node:events"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + decodeRudpPacket, + encodeRudpPacket, + type CultNetRudpPacket, +} from "../src"; + +const connectionId = 0x10203080; +const enc = (text: string) => new TextEncoder().encode(text); +const dec = (bytes: Uint8Array) => new TextDecoder().decode(bytes); +const session = (initialSequence: number) => new CultNetRudpSession({ connectionId, initialSequence }); +const send = (from: CultNetRudpSession, text: string) => from.send("schema", enc(text), { reliable: true, ordered: true }); +const names = (result: { delivered: { payload: Uint8Array }[] }) => result.delivered.map((frame) => dec(frame.payload)); + +function handshake(client: CultNetRudpSession, server: CultNetRudpSession): { connect: CultNetRudpPacket; accept: CultNetRudpPacket } { + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 0); + client.receive(accept, 0); + assert.ok(client.connected && server.connected); + return { connect, accept }; +} + +test("a reconnect on the same session is not stranded by a write the ended session owed", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + send(server, "lost"); + server.createDisconnect(enc("gone")); + + client.receive(server.acceptConnect(client.createConnect(1), 1), 1); + assert.deepEqual(names(client.receive(send(server, "after"), 2)), ["after"]); +}); + +test("a server that accepts a new client forgets the old client's sequences", () => { + const oldClient = session(50); + const server = session(500); + handshake(oldClient, server); + for (const name of ["c1", "c2", "c3"]) { + assert.deepEqual(names(server.receive(send(oldClient, name), 1)), [name]); + } + server.createDisconnect(enc("restart")); + + const newClient = session(2); + newClient.receive(server.acceptConnect(newClient.createConnect(2), 2), 2); + assert.deepEqual(names(server.receive(send(newClient, "fresh"), 3)), ["fresh"]); +}); + +test("a retransmitted Connect after data has flowed does not reset the session", () => { + const client = session(1); + const server = session(500); + const { connect } = handshake(client, server); + const frame = send(client, "once"); + assert.deepEqual(names(server.receive(frame, 1)), ["once"]); + + assert.equal(server.connectRepeats(connect), true); + assert.equal(server.acceptConnect(connect, 2).packetType, "ack", "the Accept was already acknowledged"); + assert.equal(server.outstandingReliablePacketCount, 0, "a repeat queued something"); + assert.deepEqual(server.receive(frame, 3).delivered, [], "a retransmitted Connect made the server forget what it had delivered"); +}); + +test("a Connect with another sequence is not a repeat whatever it carries", () => { + const server = session(500); + const connect = session(1).createConnect(0, enc("same payload")); + server.acceptConnect(connect, 0); + assert.equal(server.connectRepeats(connect), true); + + assert.equal(server.connectRepeats(session(9).createConnect(0, enc("same payload"))), false); + assert.equal(server.connectRepeats({ ...connect, packetType: "ping" }), false); + server.createDisconnect(); + assert.equal(server.connectRepeats(connect), false, "an ended session repeats nothing"); +}); + +test("an Accept that does not name the pending Connect is ignored", () => { + const client = session(10); + const server = session(500); + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 0); + + client.receive({ ...accept, ack: connect.sequence + 50, ackMask: 0 }, 1); + assert.equal(client.connected, false, "a stale Accept connected the session"); + assert.deepEqual(client.pendingReliableSequences, [connect.sequence]); + + client.receive({ ...accept, ack: connect.sequence + 3, ackMask: 1 << 2 }, 2); + assert.equal(client.connected, true, "the mask names the Connect"); +}); + +test("an Accept after a local disconnect does not reconnect the session", () => { + const client = session(1); + const server = session(500); + const { accept } = handshake(client, server); + client.createDisconnect(enc("bye")); + assert.equal(client.connected, false); + + client.receive(accept, 5); + assert.equal(client.connected, false, "a late Accept revived an ended session"); +}); + +test("a reconnect forgets what the old server sent", () => { + const client = session(1); + const oldServer = session(500); + handshake(client, oldServer); + for (const name of ["d1", "d2", "d3"]) { + assert.deepEqual(names(client.receive(send(oldServer, name), 1)), [name]); + } + + const newServer = session(501); + client.receive(newServer.acceptConnect(client.createConnect(2), 2), 2); + assert.deepEqual(names(client.receive(send(newServer, "fresh"), 3)), ["fresh"]); +}); + +test("an Accept for an abandoned Connect does not connect the session", () => { + const client = session(1); + const server = session(500); + const accept = server.acceptConnect(client.createConnect(0), 0); + client.createDisconnect(enc("never mind")); + + client.receive(accept, 1); + assert.equal(client.connected, false, "an Accept revived an abandoned Connect"); +}); + +test("a full queue owed to a vanished peer does not refuse a new generation", () => { + const bounded = (initialSequence: number) => new CultNetRudpSession({ connectionId, initialSequence, maxPendingReliablePackets: 1 }); + const server = bounded(500); + server.acceptConnect(session(1).createConnect(0), 0); + assert.equal(server.outstandingReliablePacketCount, 1, "the Accept is owed and never acknowledged"); + const next = session(9).createConnect(1); + server.acceptConnect(next, 1); + assert.equal(server.connectRepeats(next), true); + + const client = bounded(1); + client.createConnect(0); + client.createConnect(1); + assert.equal(client.outstandingReliablePacketCount, 1, "only the new Connect is owed"); +}); + +test("a duplicate Accept does not reseed the watermark", () => { + const client = session(1); + const server = session(500); + const { accept } = handshake(client, server); + const s1 = send(server, "s1"); + const s2 = send(server, "s2"); + client.receive({ ...accept, sequence: s2.sequence }, 1); + assert.deepEqual(client.receive(s2, 2).delivered, [], "s1 is still missing"); + assert.deepEqual(names(client.receive(s1, 3)), ["s1", "s2"]); +}); + +test("reliable data before the Accept is neither delivered nor acknowledged", () => { + const client = session(1); + const server = session(500); + const accept = server.acceptConnect(client.createConnect(0), 0); + const a = send(server, "A"); + const b = send(server, "B"); + + for (const early of [b, a]) { + assert.deepEqual(client.receive(early, 1).delivered, []); + const ack = client.createAckForReceived(early.sequence); + assert.deepEqual([ack.ack, ack.ackMask], [0, 0], "an unhandled packet was acknowledged"); + } + client.receive(accept, 2); + assert.ok(client.connected); + assert.equal(server.outstandingReliablePacketCount, 3, "nothing was acknowledged"); + + assert.deepEqual(names(client.receive(a, 3)), ["A"]); + assert.deepEqual(names(client.receive(b, 3)), ["B"]); +}); + +test("ordered frames after the Accept are delivered in order whatever arrives first", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + const a = send(server, "A"); + const b = send(server, "B"); + assert.deepEqual(client.receive(b, 1).delivered, []); + assert.deepEqual(names(client.receive(a, 2)), ["A", "B"]); +}); + +test("an unordered reliable frame is delivered on receipt, never held behind a gap", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + send(server, "lost"); + const unordered = server.send("media", enc("now"), { reliable: true, ordered: false }); + const ordered = send(server, "later"); + assert.deepEqual(names(client.receive(unordered, 1)), ["now"]); + assert.deepEqual(client.receive(ordered, 2).delivered, [], "the ordered frame waits for the gap"); +}); + +test("a peer Disconnect does not forget what the peer sent", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + const s1 = send(client, "s1"); + const s2 = send(client, "s2"); + assert.deepEqual(server.receive(s2, 1).delivered, [], "s1 is missing, so s2 is held"); + + server.receive(client.createDisconnect(enc("bye")), 2); + assert.deepEqual(names(server.receive(s1, 3)), ["s1", "s2"], "the Disconnect forgot the held frame"); + assert.deepEqual(server.receive(s2, 4).delivered, [], "the Disconnect forgot what was received"); +}); + +test("sessions from one options object each draw their own initial sequence from one to two to the thirty first", () => { + const options = { connectionId }; + const firsts = new Set(Array.from({ length: 64 }, () => new CultNetRudpSession(options).createConnect(0).sequence)); + assert.ok(firsts.size > 32, `a default session does not draw its own sequence: ${[...firsts]}`); + assert.ok([...firsts].every((value) => value >= 1 && value < 2 ** 31), `left [1, 2^31): ${[...firsts]}`); +}); + +// Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed +// Connect from an earlier attempt is stale, a Connect from a new endpoint is a +// new client. + +const namesSequence = (ack: CultNetRudpPacket, sequence: number) => + ack.ack === sequence + || Array.from({ length: 32 }, (_, bit) => bit).some( + (bit) => (ack.ackMask & (1 << bit)) !== 0 && ack.ack - bit - 1 === sequence, + ); + +test("a frame from the client's earlier generation is acknowledged, not delivered again", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const old = send(client, "old"); + assert.deepEqual(names(server.receive(old, 1)), ["old"]); + + const connect = client.createConnect(2); + client.receive(server.acceptConnect(connect, 2), 2); + assert.ok(old.sequence < connect.sequence); + assert.deepEqual(server.receive(old, 3).delivered, [], "an earlier generation's frame was delivered again"); + assert.ok(namesSequence(server.createAckForReceived(old.sequence), old.sequence), "the duplicate was not acknowledged"); + assert.deepEqual(names(server.receive(send(client, "fresh"), 4)), ["fresh"]); +}); + +test("a frame from the server's earlier generation is acknowledged, not delivered again", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const old = send(server, "old"); + assert.deepEqual(names(client.receive(old, 1)), ["old"]); + + client.receive(server.acceptConnect(client.createConnect(2), 2), 2); + assert.deepEqual(client.receive(old, 3).delivered, [], "an earlier generation's frame was delivered again"); + assert.ok(namesSequence(client.createAckForReceived(old.sequence), old.sequence)); + assert.deepEqual(names(client.receive(send(server, "fresh"), 4)), ["fresh"]); +}); + +test("a duplicate below the receive window is acknowledged by name", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const first = send(client, "first"); + assert.deepEqual(names(server.receive(first, 1)), ["first"]); + for (let index = 0; index < 4_200; index += 1) { + server.receive(send(client, "x"), 1); + client.receive(server.createAck(), 1); + } + assert.deepEqual(server.receive(first, 2).delivered, []); + assert.equal(server.createAckForReceived(first.sequence).ack, first.sequence, "a duplicate below the window was not acknowledged by name"); +}); + +test("a delayed Connect from an earlier attempt does not strand the client", () => { + const client = session(10); + const server = session(500); + const earlier = client.createConnect(0); + const retried = client.createConnect(300); + const accept = server.acceptConnect(retried, 301); + client.receive(accept, 302); + server.receive(client.createAckForReceived(accept.sequence), 302); + assert.ok(client.connected); + const a = send(client, "a"); + assert.deepEqual(names(server.receive(a, 303)), ["a"]); + client.receive(server.createAckForReceived(a.sequence), 303); + + assert.equal(server.connectRepeats(earlier), true, "the delayed Connect starts nothing"); + const reply = server.acceptConnect(earlier, 304); + assert.equal(reply.packetType, "ack"); + client.receive(reply, 305); + assert.ok(client.connected); + + const b = send(client, "b"); + assert.deepEqual(names(server.receive(b, 306)), ["b"], "the delayed Connect reset the server"); + client.receive(server.createAckForReceived(b.sequence), 307); + assert.ok(!client.pendingReliableSequences.includes(b.sequence)); +}); + +test("a repeated Connect refreshes liveness and a stale one does not", () => { + const client = session(10); + const server = session(500); + const earlier = client.createConnect(0); + const current = client.createConnect(1); + server.acceptConnect(current, 0); + + server.acceptConnect(current, 900); + assert.equal(server.checkTimeout(1_000, 500), false, "a repeated Connect did not refresh liveness"); + + server.acceptConnect(earlier, 1_400); + assert.equal(server.checkTimeout(1_600, 500), true, "a stale Connect refreshed liveness"); +}); + +test("stale Connects are recognised by serial arithmetic", () => { + const server = session(500); + server.acceptConnect(session(3).createConnect(0), 0); + const connect = (sequence: number) => session(sequence).createConnect(0); + assert.equal(server.connectRepeats(connect(2 ** 32 - 2)), true, "just before the wrap"); + assert.equal(server.connectRepeats(connect(4)), false, "just after"); + assert.equal(server.connectRepeats(connect(3 + 4_096)), false, "far after"); + assert.equal(server.connectRepeats(connect(2 ** 32 - 4_092)), true, "the window's edge"); + assert.equal(server.connectRepeats(connect(2 ** 32 - 4_093)), false, "past the window"); +}); + +test("a restarted client whose first sequence is stale connects after redrawing", () => { + const server = session(500); + const old = session(1_000); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + + const restarted = session(900); + const first = restarted.createConnect(0, enc("join")); + // The old client has not acknowledged its Accept yet, so that is the reply: + // it names the old Connect, not this one. + const early = server.acceptConnect(first, 1); + assert.equal(early.packetType, "accept"); + restarted.receive(early, 1); + assert.equal(restarted.connected, false, "an Accept for another Connect connected the client"); + server.receive(old.createAckForReceived(oldAccept.sequence), 1); + const reply = server.acceptConnect(first, 2); + assert.equal(reply.packetType, "ack"); + restarted.receive(reply, 2); + assert.equal(restarted.connected, false); + + const retransmitted = restarted.dueResends(1_000); + assert.equal(retransmitted.length, 1); + assert.equal(retransmitted[0]!.sequence, first.sequence, "the attempt is still young"); + + const fresh = restarted.dueResends(3_500); + assert.equal(fresh.length, 1); + assert.equal(fresh[0]!.packetType, "connect"); + assert.notEqual(fresh[0]!.sequence, first.sequence, "the same Connect was retransmitted for ever"); + assert.equal(dec(fresh[0]!.payload ?? new Uint8Array()), "join", "the fresh attempt lost the Connect payload"); + + const accept = server.acceptConnect(fresh[0]!, 3_500); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 3_500); + assert.ok(restarted.connected); + assert.deepEqual(names(server.receive(send(restarted, "hello"), 3_501)), ["hello"]); +}); + +test("an answered Connect is never replaced by the attempt timeout", () => { + const client = session(10); + const server = session(500); + const { connect } = handshake(client, server); + const resent = client.dueResends(60_000); + assert.ok(resent.every((packet) => packet.packetType !== "connect" || packet.sequence === connect.sequence), "a connected client started a new attempt"); + assert.ok(client.connected); +}); + +async function bind(): Promise { + const socket = createSocket("udp4"); + socket.bind(0, "127.0.0.1"); + await once(socket, "listening"); + return socket; +} +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("server mode admits a pinned client that restarts on a new port", async () => { + const serverSocket = await bind(); + const a = await bind(); + const b = await bind(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "srv", socket: serverSocket, mode: "server", connectionId, resendPollMs: 20, resendDelayMs: 20, + }); + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(dec(frame.payload))); + const port = (serverSocket.address() as { port: number }).port; + const atA: CultNetRudpPacket[] = []; + const atB: CultNetRudpPacket[] = []; + a.on("message", (wire) => atA.push(decodeRudpPacket(wire))); + b.on("message", (wire) => atB.push(decodeRudpPacket(wire))); + try { + const first = session(1); + a.send(encodeRudpPacket(first.createConnect(0, enc("join"))), port, "127.0.0.1"); + await sleep(100); + const accept = atA.find((packet) => packet.packetType === "accept"); + assert.ok(accept); + first.receive(accept, 0); + a.send(encodeRudpPacket(first.createAckForReceived(accept.sequence)), port, "127.0.0.1"); + a.send(encodeRudpPacket(send(first, "hello")), port, "127.0.0.1"); + await sleep(100); + assert.deepEqual(frames, ["hello"]); + + const second = session(1); // the process restarted on a new port, pinned + b.send(encodeRudpPacket(second.createConnect(0, enc("join"))), port, "127.0.0.1"); + await sleep(100); + const acceptB = atB.find((packet) => packet.packetType === "accept"); + assert.ok(acceptB, `the restarted client on a new port was not admitted: ${JSON.stringify(atB.map((packet) => packet.packetType))}`); + second.receive(acceptB, 1); + assert.ok(second.connected); + b.send(encodeRudpPacket(send(second, "after")), port, "127.0.0.1"); + await sleep(100); + assert.deepEqual(frames, ["hello", "after"]); + } finally { + server.close(); + a.close(); + b.close(); + } +}); + +test("an accepted peer that goes silent times out", () => { + const server = session(500); + server.acceptConnect(session(10).createConnect(0), 0); + assert.equal(server.checkTimeout(100_000, 1_000), true); +}); + +// Ack Cut 1d, batch 2: the connect-attempt timeout. + +test("an Ack naming the pending Connect does not retire it", () => { + const server = session(500); + const old = session(1); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + server.receive(old.createAckForReceived(oldAccept.sequence), 0); + + const restarted = session(1); + const first = restarted.createConnect(0, enc("join")); + const reply = server.acceptConnect(first, 1); + assert.equal(reply.packetType, "ack"); + assert.ok(namesSequence(reply, first.sequence), "the Ack must name the pending Connect for this test to bite"); + restarted.receive(reply, 1); + assert.equal(restarted.connected, false); + assert.deepEqual(restarted.pendingReliableSequences, [first.sequence], "an Ack retired the Connect"); + + const fresh = restarted.dueResends(3_000); + assert.equal(fresh.length, 1, "the client waits for ever with nothing to resend"); + assert.equal(dec(fresh[0]!.payload ?? new Uint8Array()), "join"); + const accept = server.acceptConnect(fresh[0]!, 3_000); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 3_000); + assert.ok(restarted.connected); +}); + +test("a late copy of an abandoned Connect is stale once its replacement is accepted", () => { + const server = session(500); + const client = session(10); + const abandoned = client.createConnect(0); + const fresh = client.dueResends(3_000); + assert.equal(fresh.length, 1); + assert.equal(fresh[0]!.sequence, abandoned.sequence + 4_095); + const accept = server.acceptConnect(fresh[0]!, 3_001); + client.receive(accept, 3_002); + server.receive(client.createAckForReceived(accept.sequence), 3_002); + assert.ok(client.connected); + const a = send(client, "a"); + assert.deepEqual(names(server.receive(a, 3_003)), ["a"]); + client.receive(server.createAckForReceived(a.sequence), 3_003); + + assert.equal(server.connectRepeats(abandoned), true, "the late copy would restart the server"); + const reply = server.acceptConnect(abandoned, 3_004); + assert.equal(reply.packetType, "ack"); + client.receive(reply, 3_005); + assert.ok(client.connected); + assert.deepEqual(names(server.receive(send(client, "b"), 3_006)), ["b"]); +}); + +test("a server sitting at the jump is left by the next attempt", () => { + const server = session(500); + const old = session(5_095); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + server.receive(old.createAckForReceived(oldAccept.sequence), 0); + + const restarted = session(1_000); + const first = restarted.createConnect(0, enc("join")); + restarted.receive(server.acceptConnect(first, 1), 1); + assert.equal(restarted.connected, false); + + const second = restarted.dueResends(3_000)[0]!; + assert.equal(second.sequence, 5_095, "the jump lands on the server's generation"); + const reply = server.acceptConnect(second, 3_001); + assert.equal(reply.packetType, "ack"); + restarted.receive(reply, 3_001); + assert.equal(restarted.connected, false); + + const third = restarted.dueResends(6_000)[0]!; + assert.equal(third.sequence, 5_095 + 4_095); + const accept = server.acceptConnect(third, 6_001); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 6_001); + assert.ok(restarted.connected); +}); + +test("server mode admits a pinned client that restarts on the same address", async () => { + const serverSocket = await bind(); + const peer = await bind(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "srv", socket: serverSocket, mode: "server", connectionId, resendPollMs: 20, resendDelayMs: 20, + }); + const port = (serverSocket.address() as { port: number }).port; + const received: CultNetRudpPacket[] = []; + peer.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const take = async (type: string) => { + for (let index = 0; index < 50; index += 1) { + const found = received.findIndex((packet) => packet.packetType === type); + if (found >= 0) return received.splice(found, 1)[0]!; + await sleep(10); + } + throw new Error(`no ${type} arrived`); + }; + try { + const first = session(1); + peer.send(encodeRudpPacket(first.createConnect(0, enc("join"))), port, "127.0.0.1"); + const accept = await take("accept"); + first.receive(accept, 0); + peer.send(encodeRudpPacket(first.createAckForReceived(accept.sequence)), port, "127.0.0.1"); + await sleep(50); + received.length = 0; + + const restarted = session(1); + peer.send(encodeRudpPacket(restarted.createConnect(0, enc("join"))), port, "127.0.0.1"); + restarted.receive(await take("ack"), 1); + assert.equal(restarted.connected, false); + + peer.send(encodeRudpPacket(restarted.dueResends(3_000)[0]!), port, "127.0.0.1"); + restarted.receive(await take("accept"), 3_001); + assert.ok(restarted.connected, "the restarted pinned client was never admitted"); + } finally { + server.close(); + peer.close(); + } +}); diff --git a/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts b/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts new file mode 100644 index 00000000..519fc8aa --- /dev/null +++ b/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts @@ -0,0 +1,110 @@ +// Ordered delivery has one owner: the contiguous received-through watermark. A +// frame is held for exactly one reason, a reliable sequence below it has not +// arrived, and it is delivered in the call that fills the last such gap, +// whichever channel the gap belonged to. +import assert from "node:assert/strict"; +import test from "node:test"; +import { CultNetRudpSession } from "../src"; + +const enc = (text: string) => new TextEncoder().encode(text); +const dec = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + +function handshake(): { client: CultNetRudpSession; server: CultNetRudpSession } { + const client = new CultNetRudpSession({ connectionId: 410, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId: 410, initialSequence: 500 }); + const accept = server.acceptConnect(client.createConnect(0), 0); + client.receive(accept, 0); + return { client, server }; +} + +const names = (result: { delivered: { payload: Uint8Array }[] }) => result.delivered.map((frame) => dec(frame.payload)); +const send = (session: CultNetRudpSession, channel: string, payload: string, ordered: boolean) => + session.send(channel, enc(payload), { reliable: true, ordered }); + +test("an ordered frame waits for a gap filled by another channel", () => { + const { client: sender, server: receiver } = handshake(); + const o1 = send(sender, "schema", "o1", true); + const u = send(sender, "rel", "u", false); + const o2 = send(sender, "schema", "o2", true); + const o3 = send(sender, "schema", "o3", true); + const o4 = send(sender, "schema", "o4", true); + + assert.deepEqual(names(receiver.receive(o1, 1)), ["o1"]); + assert.deepEqual(names(receiver.receive(o2, 2)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(o3, 3)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(o4, 4)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(u, 5)), ["u", "o2", "o3", "o4"]); +}); + +test("two ordered channels release each other in sequence order", () => { + const { client: sender, server: receiver } = handshake(); + const a1 = send(sender, "schema", "A1", true); + const b1 = send(sender, "other", "B1", true); + const a2 = send(sender, "schema", "A2", true); + const a3 = send(sender, "schema", "A3", true); + + const delivered = [ + ...names(receiver.receive(a1, 1)), + ...names(receiver.receive(a2, 2)), + ...names(receiver.receive(a3, 3)), + ]; + assert.deepEqual(delivered, ["A1"], "B1 is missing, so A2 and A3 wait"); + assert.deepEqual(names(receiver.receive(b1, 4)), ["B1", "A2", "A3"]); +}); + +test("a channel first used after other traffic loses nothing", () => { + const { client: sender, server: receiver } = handshake(); + const c1 = send(sender, "late", "C1", true); + const c2 = send(sender, "late", "C2", true); + const x = send(sender, "schema", "X", true); + + assert.deepEqual(names(receiver.receive(x, 1)), [], "C1 is missing"); + assert.deepEqual(names(receiver.receive(c2, 2)), [], "C1 is missing"); + assert.deepEqual(names(receiver.receive(c1, 3)), ["C1", "C2", "X"]); +}); + +test("Accept seeds the watermark of the connecting side", () => { + const { client, server } = handshake(); + const s1 = send(server, "schema", "s1", true); + const s2 = send(server, "schema", "s2", true); + + assert.deepEqual(names(client.receive(s2, 1)), []); + assert.deepEqual(names(client.receive(s1, 2)), ["s1", "s2"]); +}); + +test("Connect seeds the watermark of the accepting side", () => { + const { client, server } = handshake(); + const first = send(client, "schema", "first", true); + const second = send(client, "schema", "second", true); + + assert.deepEqual(names(server.receive(second, 1)), []); + assert.deepEqual(names(server.receive(first, 2)), ["first", "second"]); +}); + +test("a duplicate of a held frame is not delivered twice", () => { + const { client: sender, server: receiver } = handshake(); + const s1 = send(sender, "schema", "s1", true); + const s2 = send(sender, "schema", "s2", true); + + assert.deepEqual(names(receiver.receive(s2, 1)), []); + assert.deepEqual(names(receiver.receive(s2, 2)), []); + assert.deepEqual(names(receiver.receive(s1, 3)), ["s1", "s2"]); + assert.deepEqual(names(receiver.receive(s2, 4)), []); +}); + +test("resetting peer state forgets held frames and reseeds the watermark from the next handshake", () => { + const { client: sender, server: receiver } = handshake(); + send(sender, "schema", "s1", true); + const s2 = send(sender, "schema", "s2", true); + assert.deepEqual(names(receiver.receive(s2, 1)), []); + + receiver.resetPeerState(); + const next = new CultNetRudpSession({ connectionId: 410, initialSequence: 100 }); + next.receive(receiver.acceptConnect(next.createConnect(0), 2), 2); + const d1 = send(next, "schema", "d1", true); + const d2 = send(next, "schema", "d2", true); + + // The held s2 died with the reset, and the watermark follows the new peer. + assert.deepEqual(names(receiver.receive(d2, 3)), []); + assert.deepEqual(names(receiver.receive(d1, 4)), ["d1", "d2"]); +}); diff --git a/packages/cultnet-ts/test/rudp-session-ending.test.ts b/packages/cultnet-ts/test/rudp-session-ending.test.ts new file mode 100644 index 00000000..36381023 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-session-ending.test.ts @@ -0,0 +1,72 @@ +// A write belongs to the session generation it was issued in. Every way a +// generation ends drops the writes it still owed, so none is retransmitted +// into a later session where the new peer would deliver it. +import assert from "node:assert/strict"; +import test from "node:test"; +import { CultNetRudpSession } from "../src"; + +const connectionId = 0x10203070; +const enc = (text: string) => new TextEncoder().encode(text); + +function clientWithALostWrite(): { client: CultNetRudpSession; server: CultNetRudpSession } { + const client = new CultNetRudpSession({ connectionId, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId, initialSequence: 500 }); + client.receive(server.acceptConnect(client.createConnect(0), 0), 0); + client.send("schema", enc("owed to the old session"), { reliable: true, ordered: true }); + // A fragmented write larger than the send window leaves part of it queued. + client.sendMany("schema", new Uint8Array(40 * 8), { reliable: true, ordered: true, maxFragmentBytes: 8 }); + assert.equal(client.outstandingReliablePacketCount, 41); + assert.ok(client.queuedReliablePacketCount > 0); + return { client, server }; +} + +const endings: Record void> = { + "peer Disconnect": (client, server) => { + client.receive(server.createDisconnect(enc("bye")), 1); + }, + "local disconnect": (client) => { + client.createDisconnect(enc("bye")); + }, + timeout: (client) => { + assert.equal(client.checkTimeout(1_000, 10), true); + }, + reset: (client) => { + client.resetPeerState(); + }, +}; + +for (const [name, end] of Object.entries(endings)) { + test(`a ${name} drops the writes the session owed`, () => { + const { client, server } = clientWithALostWrite(); + end(client, server); + + assert.equal(client.outstandingReliablePacketCount, 0, "the write survived the end"); + + const nextServer = new CultNetRudpSession({ connectionId, initialSequence: 900 }); + nextServer.acceptConnect(client.createConnect(2_000), 2_000); + for (const resend of client.dueResends(60_000)) { + assert.deepEqual( + nextServer.receive(resend, 60_000).delivered, + [], + "the old session's write was delivered in the next session", + ); + } + }); +} + +test("an ending does not forget what was received from the peer", () => { + const client = new CultNetRudpSession({ connectionId, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId, initialSequence: 500 }); + client.receive(server.acceptConnect(client.createConnect(0), 0), 0); + const s1 = client.send("schema", enc("s1"), { reliable: true, ordered: true }); + const s2 = client.send("schema", enc("s2"), { reliable: true, ordered: true }); + assert.deepEqual(server.receive(s2, 1).delivered, [], "s1 is missing, so s2 is held"); + + assert.equal(server.checkTimeout(1_000, 10), true); + assert.deepEqual( + server.receive(s1, 1_001).delivered.map((frame) => new TextDecoder().decode(frame.payload)), + ["s1", "s2"], + "the ending forgot the held frame", + ); + assert.deepEqual(server.receive(s2, 1_002).delivered, [], "the ending forgot what was received"); +}); diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index e449aa21..a9b55c8f 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -284,7 +284,7 @@ public static class CultDocumentMessagePackSerialization !string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal)) { throw new CultStoreUnreadableException( - $"Store format {version} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}."); + $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}."); } // Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust. @@ -299,6 +299,17 @@ public static class CultDocumentMessagePackSerialization } } + // A store header as a refusal may show it: echoed in the shape cultcache.store.v, and otherwise described + // only by its length, since the bytes are the store's. + private static string DescribeHeader(string header) + { + const string prefix = "cultcache.store.v"; + var version = header.StartsWith(prefix, StringComparison.Ordinal) ? header.Substring(prefix.Length) : ""; + return version.Length > 0 && version.All(character => character >= '0' && character <= '9') + ? header + : $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes"; + } + private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) { writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount); diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 2f5755f0..e3ceeaef 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -298,8 +298,8 @@ array (a second store appended included), is not an array, has fewer or more tha (a header alone included), whose first slot is not a header it reads, or whose catalog or records it cannot decode (a record with more slots than its header allows, a variant record it does not read). A runtime never skips a slot it does not understand. It refuses on open -and on rewrite alike, because the rewrite asks the same full reader as the open. A zero-byte -file and an empty array are an empty store. A flush or a commit replaces a file only when the +and on rewrite alike, because the rewrite asks the same full reader as the open. A missing +file and an empty array are an empty store; a zero-byte file is not a store and is refused as unreadable. A flush or a commit replaces a file only when the runtime's own reader opens it, so one verdict covers open, flush and commit, and a file the runtime cannot read is left as it is. Each runtime refuses with one typed error: `CultStoreUnreadableException` (C#), `StoreUnreadableError` (Rust, TypeScript, Python), with diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index f1646299..5b0cb5a0 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -47,10 +47,20 @@ which route wrote it. - Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a per-cache, in-memory number assigned when the cache admits the change, under its gate. It increases in admission order and is not persisted. - `OnUpdate(previous, current)` carries none. Streams CultNet derives from - `OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server, - database-backed Mesh handles) have no ordering guarantee and no stale - protection. + `OnUpdate(previous, current)` carries none. CultNet derives its streams + (`CultNetDatabase.WatchAllChanges`, the subscription server, database-backed + Mesh handles) from `Watch` plus one journal (`AddJournal`), so a load, a + commit and a variant dependent's re-resolution all reach subscribers and the + mutation log. The journal runs under the gate in `Sequence` order, so the + shard log is written in cache order; the `Watch` observer publishes what the + journal decided. A database write door only hands the journal the wire message + or replicated entry for the change it admitted. A replica publishes but never + mints log sequences. A primary logs every change it admits, loads and pulls + included; a store attached after the database exists has its hydration logged + too. A primary that cannot log a change compacts through that change's + sequence and throws `CultNetShardLogException` to the writer after publication: + the commit stands, and a replica behind it resynchronizes from a snapshot. The + streams carry no `Sequence` and have no stale protection. - `GetWithSequence(key)` returns the document and the cache's current `Sequence`, read together under the gate: every change with a `Sequence` at or below it is reflected in the document. @@ -125,6 +135,10 @@ which route wrote it. needs the cache or its gate: another thread's write, a lock a writer holds. Its own leaf locks, taken only inside the journal, are the only locks that may nest under the gate. +- A shard mutation log store (`ICultNetShardMutationLogStore`) runs inside the + CultNet journal, under the cache's gate: it must not call the cache or wait + for a thread that does. A store that writes its own cache throws through the + journal's reentry guard, and the admission stands. - A thread holds one cache's gate at a time: entering a cache's hold inside another cache's hold throws. - A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 70b9ee79..cae0943c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2335,7 +2335,15 @@ namespace GameCult.Caching Remove(handle.Key); } - // false: a condition failed; nothing was written, changed in memory, or published. + /// + /// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published. + /// + /// + /// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer + /// receives the failure. One failure arrives as itself, several as an . A writer that + /// wants a particular failure (a CultNetShardLogException, when a database's log refused the commit) catches that type, + /// or unwraps and looks for it. + /// public bool Commit(Action stage) { return Land(stage, wait: true) == CultCommitOutcome.Committed; @@ -2673,7 +2681,8 @@ namespace GameCult.Caching // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not // guaranteed; each change carries the Sequence it was admitted with. Journals run before that, under the gate, // in Sequence order (AddJournal); no hold may be entered on this cache while one runs. Every change reaches every observer even - // if an observer or an OnUpdate handler throws; then the first exception (an AggregateException for several) is rethrown. + // if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their + // origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions. // If the body threw, its exception wins. internal T Held(Func body) { @@ -3984,7 +3993,7 @@ namespace GameCult.Caching } // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is - // gone or zero bytes is an empty store: nothing to read, nothing to lose. + // gone is an empty store: nothing to read, nothing to lose. A zero-byte file is not a store; it is refused. private CultPersistedStoreSnapshot? ReadSnapshot() { var bytes = ReadDisk(); @@ -4007,7 +4016,7 @@ namespace GameCult.Caching return null; try { - return ReadAllBytesShared(FileInfo.FullName) is { Length: > 0 } bytes ? bytes : null; + return ReadAllBytesShared(FileInfo.FullName); } catch (FileNotFoundException) { diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 18792413..e3f23430 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -537,14 +537,18 @@ namespace GameCult.Mesh /// public sealed class CultMeshRudpSocketOptions { - /// Gets or sets the local bind host. - public string BindHost { get; set; } = "0.0.0.0"; + /// + /// Gets or sets the local bind host. Unset, a server binds every IPv4 interface, and a + /// client binds loopback for a loopback endpoint and the unspecified address of the + /// endpoint's family otherwise, so it can reach a remote host. + /// + public string? BindHost { get; set; } /// Gets or sets the local bind port. public int BindPort { get; set; } /// Gets or sets a caller-owned bound socket. public Socket? Socket { get; set; } - /// Gets or sets the first local packet sequence. - public uint InitialSequence { get; set; } = 1; + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random. + public uint? InitialSequence { get; set; } /// Gets or sets the reliable resend delay in milliseconds. public long ResendDelayMs { get; set; } = 250; /// Gets or sets the advertised transport id. @@ -2931,7 +2935,7 @@ namespace GameCult.Mesh return new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions { RuntimeId = runtimeId, - Socket = options.Socket ?? BindRudpSocket(options.BindHost, options.BindPort), + Socket = options.Socket ?? BindRudpSocket(ResolveRudpAddress(options.BindHost ?? "0.0.0.0"), options.BindPort), Mode = CultNetRudpSocketMode.Server, ConnectionId = connectionId, InitialSequence = options.InitialSequence, @@ -2955,12 +2959,16 @@ namespace GameCult.Mesh { if (endpoint == null) throw new ArgumentNullException(nameof(endpoint)); options ??= new CultMeshRudpSocketOptions(); + var remote = ResolveRudpAddress(endpoint.Host); + var bindAddress = options.BindHost == null + ? RudpClientBindAddress(remote) + : ResolveRudpAddress(options.BindHost); return new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions { RuntimeId = runtimeId, - Socket = options.Socket ?? BindRudpSocket(options.BindHost, options.BindPort), + Socket = options.Socket ?? BindRudpSocket(bindAddress, options.BindPort), Mode = CultNetRudpSocketMode.Client, - RemoteEndPoint = new IPEndPoint(ResolveRudpAddress(endpoint.Host), endpoint.Port), + RemoteEndPoint = new IPEndPoint(remote, endpoint.Port), ConnectionId = connectionId, InitialSequence = options.InitialSequence, ResendDelayMs = options.ResendDelayMs, @@ -3232,9 +3240,21 @@ namespace GameCult.Mesh return CultNetLocal.ConnectClient(host, port, security, configureClient); } - private static Socket BindRudpSocket(string host, int port) + // A socket bound to loopback cannot send off the host on Windows, so a client keeps + // loopback only when its peer is on loopback. + private static IPAddress RudpClientBindAddress(IPAddress remote) + { + var ipv6 = remote.AddressFamily == AddressFamily.InterNetworkV6; + if (IPAddress.IsLoopback(remote)) + { + return ipv6 ? IPAddress.IPv6Loopback : IPAddress.Loopback; + } + + return ipv6 ? IPAddress.IPv6Any : IPAddress.Any; + } + + private static Socket BindRudpSocket(IPAddress address, int port) { - var address = ResolveRudpAddress(host); var socket = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); socket.Bind(new IPEndPoint(address, port)); socket.ReceiveTimeout = 20; @@ -3377,7 +3397,8 @@ namespace GameCult.Mesh if (document != null) return document; - var untyped = database.Cache.Get(key); + // Through the database, which serves only what its shards own; the cache behind it may hold more. + var untyped = await database.GetAsync(key).ConfigureAwait(false); if (untyped != null && IsSameCultDocumentSchema(untyped.GetType())) return ConvertUntypedDocument(untyped); diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index d59cd6c7..668c088e 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using GameCult.Caching; @@ -435,6 +437,10 @@ namespace GameCult.Networking /// Gets the previous document, when present. /// public object? PreviousDocument { get; } + + // The primary's own wire entry, kept for an entry this replica changed nothing for (a row its shard does not own, an + // absent key): it is relayed downstream exactly as the primary sent it, in memory and on disk alike. + internal CultNetShardLogEntryMessage? Relayed { get; set; } } /// @@ -461,6 +467,101 @@ namespace GameCult.Networking public string Reason { get; } } + /// + /// One shard-log sequence that a primary minted for a committed change and burned because it could not log it. + /// + public readonly struct CultNetBurnedSequence : IEquatable + { + /// + /// Creates a burned sequence. + /// + public CultNetBurnedSequence(string shardId, long sequence) + { + ShardId = shardId; + Sequence = sequence; + } + + /// + /// Gets the shard whose log burned the sequence. + /// + public string ShardId { get; } + + /// + /// Gets the burned shard-log sequence. + /// + public long Sequence { get; } + + /// + public bool Equals(CultNetBurnedSequence other) => Sequence == other.Sequence && string.Equals(ShardId, other.ShardId, StringComparison.Ordinal); + + /// + public override bool Equals(object? obj) => obj is CultNetBurnedSequence other && Equals(other); + + /// + public override int GetHashCode() => (StringComparer.Ordinal.GetHashCode(ShardId ?? string.Empty) * 397) ^ Sequence.GetHashCode(); + + /// + public override string ToString() => $"{ShardId}:{Sequence}"; + } + + /// + /// Raised when a database write, or a resolution, names a schema and key that no configured shard owns. A refused write + /// caches nothing. + /// + public sealed class CultNetUnownedSchemaException : InvalidOperationException + { + /// + /// Creates an unowned schema exception. + /// + public CultNetUnownedSchemaException(string schemaId, CultRecordKey key) + : base($"No shard owns schema '{schemaId}' key '{key.Value}'; the write was refused.") + { + SchemaId = schemaId; + Key = key; + } + + /// + /// Gets the schema that no shard owns. + /// + public string SchemaId { get; } + + /// + /// Gets the key of the refused write. + /// + public CultRecordKey Key { get; } + } + + /// + /// Raised to the writer when a shard primary committed changes and could not log them. The commits stand and the + /// changes were published; each shard's log burned the sequences in and compacted past them, + /// so every replica behind one resynchronizes from a snapshot. A single journal call raises one of these carrying + /// every burned sequence, whatever number of changes or shards failed. It is deliberately not an + /// : callers that read that type as a refused write must not mistake a + /// committed one for it. + /// + /// + /// The cache may still wrap this in an when something else failed in the same hold + /// (an observer, a second database's journal). A writer that wants the log failure catches this type, or unwraps + /// and looks for it. is the + /// store's failure, or an of them when several changes failed. + /// + public sealed class CultNetShardLogException : Exception + { + /// + /// Creates a shard log exception. + /// + public CultNetShardLogException(IReadOnlyList burned, Exception cause) + : base($"Committed changes could not be logged; sequences {string.Join(", ", burned.Select(entry => $"{entry.ShardId}:{entry.Sequence}"))} are burned and compacted past.", cause) + { + Burned = burned; + } + + /// + /// Gets every sequence burned by the journal call, in commit order. + /// + public IReadOnlyList Burned { get; } + } + /// /// Database-style CultNet facade over a CultCache shard set. /// @@ -472,18 +573,39 @@ namespace GameCult.Networking private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; - private readonly Dictionary _predictedDocuments = new(StringComparer.Ordinal); - private readonly Dictionary> _mutationLogs = + // Keyed by sequence, like the durable store: recording an entry twice replaces it. + private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); + // The highest sequence this primary burned per shard (a change it committed and could not log). Serves as the + // shard's compaction floor beside the store's own, so history the log cannot serve is answered compacted_history. + private readonly Dictionary _burnedThrough = new(StringComparer.Ordinal); private readonly Dictionary _appliedShardSequences = new(StringComparer.Ordinal); // CultNet typed selection, section 2: "ordinal is the sequence of the commit that last wrote - // the row". Kept on every append (AppendMutationLogEntry is the one site), keyed by (schemaId, + // the row". Kept on every append (LogPrimaryChange and RecordReplicatedEntry are the sites), keyed by (schemaId, // recordKey) so the evaluator's order and cursor never read a clock. private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); - private bool _disposed; + private readonly IDisposable _cacheChanges; + private readonly IDisposable _cacheJournal; + // What a write door of this database knows about the change it is admitting and the cache does not: the wire + // message it carries, a prediction it reconciles, the replicated log entry it applies. The journal is the only + // logger and consumes the context under the cache gate; a door registers it under the identity of the change it + // admits (the instance it writes, or the instance it removes), and the journal takes it once, for that change + // alone. A write to the same key by anyone else is a different instance and gets no context. + private readonly ConcurrentDictionary _doors = new(ReferenceIdentity.Instance); + // One gate over the mutation log, its sequence counters, the last-write map and the publication stash. Lock order: + // the cache gate, then this, then the log store's own lock. Nothing holding this calls the cache, and a log store + // must not either (ICultNetShardMutationLogStore). Publication stays outside it: a subscriber may write the cache + // from its handler. + private readonly object _logGate = new(); + // Changes the journal has decided and the Watch observer has not yet published, by cache Sequence. The journal + // runs under the cache gate before any observer, so each change is stashed before it can be published. + private readonly Dictionary _stash = new(); + // Predictions this runtime made and has not seen reconciled. Only the journal reads or writes it, under the cache gate. + private readonly HashSet _predictedDocuments = new(StringComparer.Ordinal); + private volatile bool _disposed; /// /// Creates a database surface over a CultCache instance. @@ -502,7 +624,9 @@ namespace GameCult.Networking _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); CursorKey = options.CursorKey ?? CultNetSelectionCursorKey.Random(); InitializeLogSequencesFromStore(); - _cache.OnUpdate += PublishCacheUpdate; + // The observer first, the journal second: a change the journal stashed always finds its publisher. + _cacheChanges = _cache.Watch().Subscribe(Publish); + _cacheJournal = _cache.AddJournal(Journal); } /// @@ -536,7 +660,7 @@ namespace GameCult.Networking { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - return ResolveShardInternal(descriptor, key).IsPrimary; + return FindShard(descriptor, key)?.IsPrimary ?? false; } /// Gets whether this database instance can submit a prediction for the document at a key. @@ -557,18 +681,21 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - if (!_mutationLogs.TryGetValue(shardId, out var entries)) + lock (_logGate) { - return Array.Empty(); - } + if (!_mutationLogs.TryGetValue(shardId, out var entries)) + { + return Array.Empty(); + } - var query = entries.Where(entry => entry.Sequence > afterSequence); - if (limit.HasValue) - { - query = query.Take(limit.Value); - } + var query = entries.Values.Where(entry => entry.Sequence > afterSequence); + if (limit.HasValue) + { + query = query.Take(limit.Value); + } - return query.ToArray(); + return query.ToArray(); + } } /// @@ -598,7 +725,15 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _mutationLogStore?.GetCompactedThrough(shardId) ?? 0; + lock (_logGate) + return CompactedFloor(shardId); + } + + // The store's compaction point or this primary's own burned sequences, whichever is higher. Under _logGate. + private long CompactedFloor(string shardId) + { + var burned = _burnedThrough.TryGetValue(shardId, out var through) ? through : 0; + return Math.Max(burned, _mutationLogStore?.GetCompactedThrough(shardId) ?? 0); } /// @@ -608,11 +743,13 @@ namespace GameCult.Networking { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - var compactedThrough = GetCompactedMutationLogSequence(shardId); - var retained = _mutationLogStore != null - ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) - : GetMutationLog(shardId).Select(entry => entry.Sequence); - return retained.DefaultIfEmpty(compactedThrough).Max(); + lock (_logGate) + { + var retained = _mutationLogStore != null + ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) + : GetMutationLog(shardId).Select(entry => entry.Sequence); + return retained.Append(CompactedFloor(shardId)).Max(); + } } /// @@ -629,6 +766,10 @@ namespace GameCult.Networking ThrowIfDisposed(); if (shard == null) throw new ArgumentNullException(nameof(shard)); + // The log sequence is read before any document: a write that lands in between is then replayed from the log on + // top of documents that already hold it (whole-document puts and deletes, applied in order), and a snapshot can + // never claim a sequence whose write its documents lack. + var logSequence = GetLatestMutationLogSequence(shard.ShardId); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); var lowKeys = CultNetV0SelectionLowering.Lower(filter?.RecordKeys); @@ -642,7 +783,7 @@ namespace GameCult.Networking Documents = Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -665,7 +806,7 @@ namespace GameCult.Networking Documents = page.Documents ?? Array.Empty(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, - ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) + ShardLogSequence = logSequence }; } @@ -700,39 +841,17 @@ namespace GameCult.Networking var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { - RemoveTrackedDocument(existing.DocumentType, existing.Key); - PublishUntyped( - existing.DocumentType, - CultNetDatabaseChangeKind.Removed, - existing.Key, - existing.SchemaId, - shard, - document: null, - previousDocument: existing.Document); + using (Door(existing.Document, new DoorContext(shard) { NoLog = true, Removal = true })) + _cache.Remove(existing.Key); } foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document))) { var key = new CultRecordKey(document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(document); - var previous = _cache.Get(key); - var applied = await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document, - ShardId = shard.ShardId, - ShardEpoch = shard.Epoch - }).ConfigureAwait(false); - PublishUntyped( - descriptor.DocumentType, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - applied, - previous); + var applied = _documents.DeserializeRawDocument(document); + using (Door(applied, new DoorContext(shard) { NoLog = true })) + await _cache.UpsertAsync(descriptor.DocumentType, applied, key).ConfigureAwait(false); } var sequence = response.ShardLogSequence ?? 0; @@ -792,6 +911,7 @@ namespace GameCult.Networking /// /// Resolves the shard that governs the supplied schema and key. /// + /// No configured shard owns the schema and key. public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key) { ThrowIfDisposed(); @@ -804,16 +924,18 @@ namespace GameCult.Networking public Task GetAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); - return Task.FromResult(_cache.Get(key)); + var document = _cache.Get(key); + return Task.FromResult(document != null && Owns(_cache.Registry.GetRequired(document.GetType()), key) ? document : null); } /// - /// Gets all documents assignable to the requested type. + /// Gets all documents assignable to the requested type. A database serves only what its shards own: a cached row no + /// shard owns is left out here, as it is from every read and projection. The cache itself still holds it. /// public IEnumerable GetAll() where T : class { ThrowIfDisposed(); - return _cache.GetAll(); + return _cache.GetAll().Where(Serves); } /// @@ -822,7 +944,8 @@ namespace GameCult.Networking public T? GetByName(string name) where T : class { ThrowIfDisposed(); - return _cache.GetByName(name); + var document = _cache.GetByName(name); + return document != null && Serves(document) ? document : null; } /// @@ -831,47 +954,30 @@ namespace GameCult.Networking public T? GetByIndex(string alias, string value) where T : class { ThrowIfDisposed(); - return _cache.GetByIndex(alias, value); + var document = _cache.GetByIndex(alias, value); + return document != null && Serves(document) ? document : null; } /// /// Adds or replaces a document at a specific key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task> PutAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(document.GetType()); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - void PublishCommittedPut() - { - AppendMutationLogEntry( - shard, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - document, - previous); - Publish(new CultNetDatabaseChange( - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - document, - previous)); - } - PublishCommittedPut(); - return handle; + return await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); } /// /// Applies a locally predicted document for a client-owned input scope. /// + /// No shard owns the schema and key; nothing was written. public async Task> PutPredictedAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); @@ -881,59 +987,38 @@ namespace GameCult.Networking var shard = ResolveShardInternal(descriptor, key); EnsureClientAuthority(descriptor, key); - var previous = _cache.Get(key); - var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Predicted, - key, - descriptor.SchemaId, - shard, - document, - previous)); + CultRecordHandle handle; + using (Door(document, new DoorContext(shard) + { + Kind = CultNetDatabaseChangeKind.Predicted, + NoLog = true, + PredictionKey = PredictionKey(descriptor.SchemaId, key), + Predicts = true + })) + handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); return handle; } /// /// Deletes a document by key. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task DeleteAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); - var shard = ResolveShardInternal(descriptor, key); - EnsurePrimary(shard, descriptor.SchemaId, key); - - var previous = _cache.Get(key); - if (previous != null) - { - _cache.Remove(new CultRecordHandle(key)); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous); - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous)); - } - PublishCommittedDelete(); - } + EnsurePrimary(ResolveShardInternal(descriptor, key), descriptor.SchemaId, key); + _cache.Remove(new CultRecordHandle(key)); return Task.CompletedTask; } /// /// Applies a raw document mutation after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public async Task ApplyPutAsync(CultNetDocumentPutRawMessage message) { ThrowIfDisposed(); @@ -947,48 +1032,17 @@ namespace GameCult.Networking var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); - var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; - var predictionKey = PredictionKey(descriptor.SchemaId, key); - var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); - if (reconcilesPrediction) - { - kind = CultNetDatabaseChangeKind.Reconciled; - } - - void PublishCommittedPut() - { - if (reconcilesPrediction) - _predictedDocuments.Remove(predictionKey); - AppendMutationLogEntry( - shard, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - key, - descriptor.SchemaId, - document, - previous, - new CultNetShardLogEntryMessage - { - ChangeKind = previous == null ? "added" : "updated", - Put = message - }); - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); - } - PublishCommittedPut(); + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Put = message, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); return document; } /// /// Applies a raw document delete after checking shard authority. /// + /// No shard owns the schema and key; nothing was written. + /// The change committed and was published, but the shard log could not record it. public Task ApplyDeleteAsync(CultNetDocumentDeleteMessage message) { ThrowIfDisposed(); @@ -1004,31 +1058,8 @@ namespace GameCult.Networking return Task.CompletedTask; } - _cache.Remove(key); - void PublishCommittedDelete() - { - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous, - wireEntry: new CultNetShardLogEntryMessage - { - ChangeKind = "removed", - Delete = message - }); - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); - } - PublishCommittedDelete(); + using (Door(previous, new DoorContext(shard) { Delete = message, Removal = true })) + _cache.Remove(key); return Task.CompletedTask; } @@ -1142,7 +1173,6 @@ namespace GameCult.Networking (change.Document != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.Document)) || (change.PreviousDocument != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.PreviousDocument))); } - /// /// Releases database subscriptions. /// @@ -1154,40 +1184,318 @@ namespace GameCult.Networking } _disposed = true; - _cache.OnUpdate -= PublishCacheUpdate; + // The journal first: disposing it waits for a journal in flight, so nothing is stashed after this line. What a + // writer has logged and not yet published is then published here, so a change the log holds is never dropped. + _cacheJournal.Dispose(); + _cacheChanges.Dispose(); + Publication[] unpublished; + lock (_logGate) + { + unpublished = _stash.OrderBy(entry => entry.Key).Select(entry => entry.Value).ToArray(); + _stash.Clear(); + } + + foreach (var publication in unpublished) + { + Publish(publication); + } + _changes.Dispose(); } - private void PublishCacheUpdate(object? previous, object? current) + // The cache is the one owner of "a change was committed"; the journal is this database's one decision point and its + // one logger. It runs under the cache gate, in cache Sequence order, before any observer: it takes the change's door, + // settles the change kind and the prediction bookkeeping, logs the change when this runtime is the shard's primary + // (a replica records only the primary's own entry, which its door carries), and stashes what the publisher needs. + // A change with no door was written by someone else (a bare cache write, a load, a pull) and is logged like any + // unattributed commit. It never enters the cache and calls nothing that does. + private void Journal(IReadOnlyList> changes) { - var document = current ?? previous; - if (document == null) + if (_disposed) { return; } - var documentType = document.GetType(); - var descriptor = _cache.Registry.GetRequired(documentType); - var key = current != null - ? GetTrackedKey(current, documentType) - : new CultRecordKey(string.Empty); - var shard = ResolveShardInternal(descriptor, key); - var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); - var change = Activator.CreateInstance( - changeType, - current == null ? CultNetDatabaseChangeKind.Removed : - previous == null ? CultNetDatabaseChangeKind.Added : - CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - current, - previous, - null); - if (change != null) + List? failures = null; + List? burned = null; + List? burnCauses = null; + foreach (var change in changes) { - _changes.OnNext(change); + var identity = change.Document ?? change.PreviousDocument; + if (identity == null) + { + continue; + } + + var door = TakeDoor(change); + var documentType = identity.GetType(); + var descriptor = _cache.Registry.GetRequired(documentType); + var shard = door?.Shard ?? FindShard(descriptor, change.Key); + if (shard == null) + { + // No shard owns a bare cache write of this schema: the database neither logs nor publishes it. + continue; + } + + var kind = door?.Kind ?? change.Kind switch + { + CultCacheDocumentChangeKind.Removed => CultNetDatabaseChangeKind.Removed, + CultCacheDocumentChangeKind.Added => CultNetDatabaseChangeKind.Added, + _ => CultNetDatabaseChangeKind.Updated + }; + if (door?.PredictionKey is { } predictionKey) + { + if (door.Predicts) + _predictedDocuments.Add(predictionKey); + else if (_predictedDocuments.Remove(predictionKey)) + kind = CultNetDatabaseChangeKind.Reconciled; + } + + // The log records a reconciled change as the update it is. + var logKind = kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind; + try + { + if (door?.Replica is { } replicated) + { + RecordReplicatedEntry(shard, logKind, descriptor.SchemaId, change, replicated); + } + else if (door?.NoLog != true && shard.IsPrimary) + { + if (LogPrimaryChange(shard, logKind, descriptor.SchemaId, change, door) is { } burn) + { + (burned ??= new List()).Add(new CultNetBurnedSequence(shard.ShardId, burn.Sequence)); + (burnCauses ??= new List()).Add(burn.Cause); + } + } + } + catch (Exception exception) + { + (failures ??= new List()).Add(exception); + } + + lock (_logGate) + _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); } + + // Every change is stashed and will be published; the writer hears of a failed log afterwards, in one exception. + if (burned != null) + { + (failures ??= new List()).Insert( + 0, + new CultNetShardLogException(burned, burnCauses!.Count == 1 ? burnCauses[0] : new AggregateException(burnCauses))); + } + + if (failures != null) + { + throw failures.Count == 1 ? failures[0] : new AggregateException(failures); + } + } + + // The Watch observer publishes, on the writer's thread after the cache gate opens and before the write returns, what + // the journal stashed for this change. A change with no stash was admitted before this database existed. + private void Publish(CultCacheDocumentChange change) + { + Publication? publication; + lock (_logGate) + { + if (!_stash.Remove(change.Sequence, out publication)) + { + return; + } + } + + Publish(publication); + } + + private void Publish(Publication publication) => + PublishUntyped( + publication.DocumentType, + publication.Kind, + publication.Key, + publication.SchemaId, + publication.Shard, + publication.Document, + publication.PreviousDocument); + + private void RecordReplicatedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + CultNetShardLogEntryMessage replicated) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + replicated.Sequence, + replicated.CommittedAt, + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument), + replicated); + _lastWriteSequence[(schemaId, change.Key.Value)] = replicated.Sequence; + } + } + + // A primary mints the sequence, appends durably, then records in memory. If it cannot log the change, the change is + // still committed: the sequence is burned and compacted past, so a replica behind it is told compacted_history and + // resynchronizes by snapshot. The burn is returned for the journal to tell the writer in one CultNetShardLogException. + private (long Sequence, Exception Cause)? LogPrimaryChange( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind logKind, + string schemaId, + CultCacheDocumentChange change, + DoorContext? door) + { + lock (_logGate) + { + var sequence = NextMutationLogSequence(shard.ShardId); + _lastWriteSequence[(schemaId, change.Key.Value)] = sequence; + try + { + var entry = new CultNetShardMutationLogEntry( + shard.ShardId, + shard.Epoch, + sequence, + DateTimeOffset.UtcNow.ToString("O"), + logKind, + schemaId, + change.Key, + change.Document, + change.PreviousDocument); + // A door's own wire message is what the log stores (its kind, sequence and shard are restated from the entry). + var wireEntry = _mutationLogStore == null + ? null + : door is { Put: not null } or { Delete: not null } + ? new CultNetShardLogEntryMessage { Put = door.Put, Delete = door.Delete } + : ToLogEntryMessage(entry); + RecordMutationLogEntry(entry, wireEntry); + return null; + } + catch (Exception cause) + { + _burnedThrough[shard.ShardId] = Math.Max(_burnedThrough.TryGetValue(shard.ShardId, out var burned) ? burned : 0, sequence); + try + { + _mutationLogStore?.CompactThrough(shard.ShardId, sequence); + } + catch (Exception compactCause) + { + cause = new AggregateException(cause, compactCause); + } + + return (sequence, cause); + } + } + } + + // A door's context belongs to one specific change, recognised by shape: a put door's change carries the instance as + // its new document; a removal door removes an instance, so its change carries it as the previous document and no + // new one. A plain removal of an instance a put door admitted is neither, and gets no context. A door's own change + // is journaled first, in its own hold, and takes its context there. + private DoorContext? TakeDoor(CultCacheDocumentChange change) + { + var identity = change.Document ?? change.PreviousDocument; + if (identity == null || !_doors.TryGetValue(identity, out var door)) + { + return null; + } + + var matches = change.Document == null + ? door.Removal + : !door.Removal; + return matches && ((ICollection>)_doors) + .Remove(new KeyValuePair(identity, door)) + ? door + : null; + } + + private DoorRegistration Door(object identity, DoorContext context) => + new(_doors, identity, context, _doors.TryAdd(identity, context)); + + // Cleanup only: a door whose change never arrived (a refused write) must not leave its context behind. + private readonly struct DoorRegistration : IDisposable + { + private readonly ConcurrentDictionary _doors; + private readonly object _identity; + private readonly DoorContext _context; + private readonly bool _owned; + + public DoorRegistration(ConcurrentDictionary doors, object identity, DoorContext context, bool owned) + { + _doors = doors; + _identity = identity; + _context = context; + _owned = owned; + } + + // True when the journal took the context: the door's write was admitted. + public bool Consumed => !_owned || !_doors.TryGetValue(_identity, out var held) || !ReferenceEquals(held, _context); + + public void Dispose() + { + if (_owned) + ((ICollection>)_doors).Remove(new KeyValuePair(_identity, _context)); + } + } + + // What the publisher needs of a change the journal decided. + private sealed class Publication + { + public Publication( + Type documentType, + CultNetDatabaseChangeKind kind, + CultRecordKey key, + string schemaId, + CultNetShardDescriptor shard, + object? document, + object? previousDocument) + { + DocumentType = documentType; + Kind = kind; + Key = key; + SchemaId = schemaId; + Shard = shard; + Document = document; + PreviousDocument = previousDocument; + } + + public Type DocumentType { get; } + public CultNetDatabaseChangeKind Kind { get; } + public CultRecordKey Key { get; } + public string SchemaId { get; } + public CultNetShardDescriptor Shard { get; } + public object? Document { get; } + public object? PreviousDocument { get; } + } + + private sealed class DoorContext + { + public DoorContext(CultNetShardDescriptor shard) => Shard = shard; + + public CultNetShardDescriptor Shard { get; } + public CultNetDatabaseChangeKind? Kind { get; set; } + public bool NoLog { get; set; } + public bool Removal { get; set; } + public CultNetDocumentPutRawMessage? Put { get; set; } + public CultNetDocumentDeleteMessage? Delete { get; set; } + public CultNetShardLogEntryMessage? Replica { get; set; } + // The prediction this door's change either records (Predicts) or ends. + public string? PredictionKey { get; set; } + public bool Predicts { get; set; } + } + + private sealed class ReferenceIdentity : IEqualityComparer + { + public static readonly ReferenceIdentity Instance = new(); + + bool IEqualityComparer.Equals(object? x, object? y) => ReferenceEquals(x, y); + + int IEqualityComparer.GetHashCode(object value) => RuntimeHelpers.GetHashCode(value); } private CultRecordKey GetTrackedKey(object document, Type documentType) @@ -1240,16 +1548,6 @@ namespace GameCult.Networking } } - private void RemoveTrackedDocument(Type documentType, CultRecordKey key) - { - _cache.Remove(key); - } - - private void Publish(CultNetDatabaseChange change) where T : class - { - _changes.OnNext(change); - } - private sealed class TrackedShardDocument { public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document) @@ -1266,41 +1564,17 @@ namespace GameCult.Networking public object Document { get; } } - private void AppendMutationLogEntry( - CultNetShardDescriptor shard, - CultNetDatabaseChangeKind kind, - CultRecordKey key, - string schemaId, - object? document, - object? previousDocument, - CultNetShardLogEntryMessage? wireEntry = null) - { - var sequence = NextMutationLogSequence(shard.ShardId); - var committedAt = DateTimeOffset.UtcNow.ToString("O"); - var entry = new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - sequence, - committedAt, - kind, - schemaId, - key, - document, - previousDocument); - var storedWireEntry = _mutationLogStore == null - ? null - : wireEntry ?? ToLogEntryMessage(entry); - RecordMutationLogEntry(entry, storedWireEntry); - _lastWriteSequence[(schemaId, key.Value)] = sequence; - } /// /// The shard-log sequence of the commit that last wrote this row, or null when the row has /// never been committed through this database. The order the CultNet selection evaluator /// pages by (docs/cultnet-selection-cut.md, section 2). /// - public long? LastWriteSequence(string schemaId, CultRecordKey key) => - _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + public long? LastWriteSequence(string schemaId, CultRecordKey key) + { + lock (_logGate) + return _lastWriteSequence.TryGetValue((schemaId, key.Value), out var sequence) ? sequence : null; + } /// /// The current write-sequence watermark across every shard this database holds, 0 when nothing @@ -1311,7 +1585,11 @@ namespace GameCult.Networking /// another and can report a page as exact "as of" a sequence another, unrelated shard advanced to /// while the page's own shard sat still. /// - public ulong CurrentAsOf() => _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + public ulong CurrentAsOf() + { + lock (_logGate) + return _lastWriteSequence.Count == 0 ? 0UL : (ulong)_lastWriteSequence.Values.Max(); + } /// /// The write-sequence watermark of one shard's own log (S-9): the sequence of the last commit @@ -1325,7 +1603,8 @@ namespace GameCult.Networking public ulong CurrentAsOf(string shardId) { if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); - return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; + lock (_logGate) + return _nextLogSequences.TryGetValue(shardId, out var next) ? (ulong)(next - 1) : 0UL; } private void RecordMutationLogEntry( @@ -1334,21 +1613,22 @@ namespace GameCult.Networking { if (!_mutationLogs.TryGetValue(entry.ShardId, out var entries)) { - entries = new List(); + entries = new SortedList(); _mutationLogs[entry.ShardId] = entries; } - entries.Add(entry); + // Disk first: memory never serves an entry the durable log lacks, and a refused append records nothing. + if (_mutationLogStore != null && wireEntry != null) + { + _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); + } + + entries[entry.Sequence] = entry; if (!_nextLogSequences.TryGetValue(entry.ShardId, out var next) || next <= entry.Sequence) { _nextLogSequences[entry.ShardId] = entry.Sequence + 1; } - - if (_mutationLogStore != null && wireEntry != null) - { - _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); - } } private long NextMutationLogSequence(string shardId) @@ -1371,12 +1651,23 @@ namespace GameCult.Networking { // R-Q: the log is the durable record of every commit, replica-applied ones included, so // the last-write sequence a restart resumes with is rebuilt from it exactly as - // AppendMutationLogEntry/the replica apply paths maintain it live - a row's ordinal for + // LogPrimaryChange/the replica apply paths maintain it live - a row's ordinal for // the selection evaluator's order and cursor must not reset to "never written" on restart. - var entries = _mutationLogStore.Read(shard.ShardId); + var entries = _mutationLogStore.Read(shard.ShardId).OrderBy(entry => entry.Sequence).ToArray(); + var compacted = _mutationLogStore.GetCompactedThrough(shard.ShardId); var highest = 0L; + var expected = compacted + 1; + var hole = 0L; foreach (var entry in entries) { + // A sequence the log lacks between two it holds was burned (its append and its compaction both failed, or + // a compaction was interrupted). Replicas must be told compacted_history for it, not served a gapped log. + if (entry.Sequence > compacted && entry.Sequence > expected) + { + hole = entry.Sequence - 1; + } + + expected = Math.Max(expected, entry.Sequence + 1); if (entry.Sequence > highest) { highest = entry.Sequence; @@ -1389,6 +1680,13 @@ namespace GameCult.Networking } } + // Sequences at or below the compaction point are spent even when the log holds none of them. + highest = Math.Max(highest, compacted); + if (hole > 0) + { + _burnedThrough[shard.ShardId] = hole; + } + if (highest > 0) { _nextLogSequences[shard.ShardId] = highest + 1; @@ -1453,8 +1751,6 @@ namespace GameCult.Networking var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document); - var previous = _cache.Get(key); - var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { @@ -1462,36 +1758,18 @@ namespace GameCult.Networking $"Shard '{shard.ShardId}' log entry {entry.Sequence} has a removed kind with a put payload."); } - var predictionKey = PredictionKey(descriptor.SchemaId, key); - if (_predictedDocuments.Remove(predictionKey)) + // The replica's log records the primary's own entry under the primary's own sequence, in the same hold that + // admits the change; the journal does both. A retry after a refused append is a fresh instance, so it matches + // its door again and re-records the entry. + if (!shard.Matches(descriptor, key)) { - kind = CultNetDatabaseChangeKind.Reconciled; + RecordUnappliedEntry(shard, kind, descriptor.SchemaId, key, entry); + return; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - descriptor.SchemaId, - key, - document, - previous), - entry); - // R-Q: a replica apply is a commit path too - the last-write sequence this row now carries - // is the entry's own sequence, exactly like AppendMutationLogEntry's local-commit path sets - // it, so the selection evaluator's order and cursor agree whether a row's most recent write - // landed locally or arrived through replication. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); + var document = _documents.DeserializeRawDocument(message.Document); + using (Door(document, new DoorContext(shard) { Kind = kind, Replica = entry, PredictionKey = PredictionKey(descriptor.SchemaId, key) })) + await _cache.UpsertAsync(descriptor.DocumentType, document, key).ConfigureAwait(false); } private void ApplyCommittedDelete( @@ -1510,37 +1788,53 @@ namespace GameCult.Networking var key = new CultRecordKey(message.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); + if (!shard.Matches(descriptor, key)) + { + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + return; + } + + var admitted = false; var previous = _cache.Get(key); if (previous != null) { + using var door = Door(previous, new DoorContext(shard) { Kind = CultNetDatabaseChangeKind.Removed, Replica = entry, Removal = true }); _cache.Remove(key); + admitted = door.Consumed; } - RecordMutationLogEntry(new CultNetShardMutationLogEntry( - shard.ShardId, - shard.Epoch, - entry.Sequence, - entry.CommittedAt, - CultNetDatabaseChangeKind.Removed, - descriptor.SchemaId, - key, - document: null, - previousDocument: previous), - entry); - // R-Q: see ApplyCommittedPutAsync - a replica-applied delete is a commit too. - _lastWriteSequence[(descriptor.SchemaId, key.Value)] = entry.Sequence; - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); + if (!admitted) + { + // The key is not here, so nothing was admitted to log. + RecordUnappliedEntry(shard, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, entry); + } + } + + // An entry that changed nothing here (an absent key, or a row this shard does not own - the snapshot path leaves such + // rows out too). This replica's log must still hold every entry of the primary's, or a replica chained on it sees a gap. + private void RecordUnappliedEntry( + CultNetShardDescriptor shard, + CultNetDatabaseChangeKind kind, + string schemaId, + CultRecordKey key, + CultNetShardLogEntryMessage entry) + { + lock (_logGate) + { + RecordMutationLogEntry(new CultNetShardMutationLogEntry( + shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, kind, + schemaId, key, document: null, previousDocument: null) { Relayed = entry }, entry); + _lastWriteSequence[(schemaId, key.Value)] = entry.Sequence; + } } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) { + if (entry.Relayed != null) + { + return NormalizeWireLogEntry(entry, entry.Relayed); + } + if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null) { return new CultNetShardLogEntryMessage @@ -1637,20 +1931,6 @@ namespace GameCult.Networking }; } - private void EnsureClientAuthority(string schemaId, CultRecordKey key) - { - if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, schemaId, key))) - { - return; - } - - var shard = ResolveShardInternal(schemaId, key); - throw new CultNetShardAuthorityException( - shard, - $"Runtime '{_runtimeId}' does not have client prediction authority for schema '{schemaId}' key '{key.Value}'.", - "not_client_authority"); - } - private void EnsureClientAuthority(CultDocumentDescriptor descriptor, CultRecordKey key) { if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, descriptor, key))) @@ -1695,15 +1975,30 @@ namespace GameCult.Networking } } - private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) + // No shard owns a schema and key that no descriptor matches, and nothing is written to one: every write path refuses. + private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? throw new CultNetUnownedSchemaException(schemaId, key); + + private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) => + FindShard(descriptor, key) ?? throw new CultNetUnownedSchemaException(descriptor.SchemaId, key); + + internal bool Owns(CultDocumentDescriptor descriptor, CultRecordKey key) => FindShard(descriptor, key) != null; + + // Whether the database serves this cached instance: a shard owns its schema and key. A shard that owns everything makes + // the lookup of the instance's key unnecessary. + private bool Serves(object document) { - return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0]; + if (_shards.Any(shard => shard.SchemaIds.Count == 0 && string.IsNullOrEmpty(shard.KeyPrefix))) + { + return true; + } + + var type = document.GetType(); + return Owns(_cache.Registry.GetRequired(type), GetTrackedKey(document, type)); } - private CultNetShardDescriptor ResolveShardInternal(CultDocumentDescriptor descriptor, CultRecordKey key) - { - return _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)) ?? _shards[0]; - } + private CultNetShardDescriptor? FindShard(CultDocumentDescriptor descriptor, CultRecordKey key) => + _shards.FirstOrDefault(shard => shard.Matches(descriptor, key)); private static void EnsurePrimary(CultNetShardDescriptor shard, string schemaId, CultRecordKey key, long? expectedEpoch = null) { diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index d53315ed..77b03070 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -112,10 +112,13 @@ namespace GameCult.Networking request); } + // A database serves only what its shards own. return _database.Documents.CreateRawSnapshotResponse( _database.Cache, string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId, - request); + request, + options: null, + rowFilter: _database.Owns); } /// @@ -135,6 +138,9 @@ namespace GameCult.Networking // R-Q: asOf is one shard-log watermark, so a selection whose matched rows span more than // one shard's log is refused rather than answered against a watermark that is not // exact for all of them. + // A row whose schema no shard owns is not part of what this database serves: it is left out, not an error. + options: null, + rowFilter: _database.Owns, shardIdOf: (schemaId, key) => _database.ResolveShard(schemaId, key).ShardId, cursorKey: _database.CursorKey, // S-9: once the matched rows resolve to one shard, asOf is that shard's own watermark, @@ -312,6 +318,16 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + // The write committed and was published; only its shard log failed, so the peer is not told it was refused. + _server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { _server.Logger.LogError($"CultNet raw put failed: {ex.Message}"); @@ -332,6 +348,15 @@ namespace GameCult.Networking peer.SendCultNet(CreateRoutingError(ex)); } } + catch (CultNetUnownedSchemaException ex) + { + _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); + peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + } + catch (Exception ex) when (CommittedWithLogFailure(ex)) + { + _server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}"); + } catch (Exception ex) { peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); @@ -604,6 +629,12 @@ namespace GameCult.Networking : messageId; } + // A CultNetShardLogException means the write committed and was published (the writer hears of a failed log afterwards); + // the cache wraps it in an AggregateException when something else failed in the same hold. + private static bool CommittedWithLogFailure(Exception exception) => + exception is CultNetShardLogException || + exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException); + private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception) { return new CultNetErrorMessage diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index f3150740..7ae3c67e 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -421,7 +421,8 @@ namespace GameCult.Networking documentSelection, ordinalOf: (schemaId, key) => _database.LastWriteSequence(schemaId, key) ?? 0, asOf: _database.CurrentAsOf(), - options: null); + options: null, + rowFilter: _database.Owns); // The snapshot is one row per record key by construction, so the duplicate-key throw this // used to guard can no longer fire (docs/cultnet-selection-cut.md, S2-2); deleted rather // than tested around. diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 805312a3..58b7ca36 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -301,7 +301,16 @@ namespace GameCult.Networking CultCache cache, string messageId, CultNetSnapshotRequestMessage? filter = null, - CultNetDocumentMessageOptions? options = null) + CultNetDocumentMessageOptions? options = null) => + CreateRawSnapshotResponse(cache, messageId, filter, options, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse( + CultCache cache, + string messageId, + CultNetSnapshotRequestMessage? filter, + CultNetDocumentMessageOptions? options, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); var lowSchemas = CultNetV0SelectionLowering.Lower(filter?.SchemaIds); @@ -344,7 +353,7 @@ namespace GameCult.Networking ordinalOf = static (_, _) => 0; } - var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options); + var page = SelectAll(cache, selection, ordinalOf, asOf: 0, options, rowFilter); return new CultNetSnapshotResponseRawMessage { @@ -369,13 +378,27 @@ namespace GameCult.Networking CultNetDocumentMessageOptions? options = null, Func? shardIdOf = null, CultNetSelectionCursorKey? cursorKey = null, - Func? asOfForShard = null) + Func? asOfForShard = null) => + CreateSelectionResponse(cache, messageId, selection, ordinalOf, asOf, options, shardIdOf, cursorKey, asOfForShard, rowFilter: null); + + // rowFilter leaves out the cached rows it rejects before evaluation (a database serves only what its shards own). + internal CultNetSnapshotResponseRawV1Message CreateSelectionResponse( + CultCache cache, + string messageId, + CultNetSelection selection, + Func ordinalOf, + ulong asOf, + CultNetDocumentMessageOptions? options, + Func? shardIdOf, + CultNetSelectionCursorKey? cursorKey, + Func? asOfForShard, + Func? rowFilter) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (selection == null) throw new ArgumentNullException(nameof(selection)); if (ordinalOf == null) throw new ArgumentNullException(nameof(ordinalOf)); - var page = SelectPage(cache, selection, ordinalOf, asOf, options, shardIdOf: shardIdOf, cursorKey: cursorKey, asOfForShard: asOfForShard); + var page = SelectPage(cache, selection, ordinalOf, asOf, options, rowFilter, shardIdOf, cursorKey, asOfForShard); return new CultNetSnapshotResponseRawV1Message { MessageId = RequireNonEmpty(messageId, nameof(messageId)), diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index 41a2017d..e19fccc3 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -43,9 +43,10 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } = 0x43554c54; /// - /// Gets or sets the first local packet sequence for each accepted peer. + /// Gets or sets the first local packet sequence for each accepted peer. Unset, each peer's + /// session draws its own at random. /// - public uint InitialSequence { get; set; } = 100; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 4daa76d5..e7f4cfd9 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -511,7 +511,7 @@ namespace GameCult.Networking [Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; } /// /// Gets or sets the machine-readable refusal code (R-N), e.g. selection_invalid, - /// cursor_stale, cursor_invalid, reference_outside_target. Additive to + /// cursor_stale, cursor_invalid, reference_outside_target, unowned_schema. Additive to /// cultnet.error.v0; a v0 peer that does not know this field ignores it. /// [Key("code")] public string? Code { get; set; } @@ -532,6 +532,17 @@ namespace GameCult.Networking Details = new CultNetErrorDetails { Field = ex.Field, Value = ex.Value } }; + /// + /// Builds the wire error for a write no shard owns: code unowned_schema, details { field: "schemaId", value }. + /// + public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) => + new CultNetErrorMessage + { + Error = $"unowned_schema: {ex.Message}", + Code = "unowned_schema", + Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId } + }; + /// /// Builds the wire error for a cursor refusal (R-N): code cursor_stale with details /// { asOf, current }, or code cursor_invalid with no details. diff --git a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs index 5b3a9c32..4c93e000 100644 --- a/src/GameCult.Networking/CultNetShardMutationLogStorage.cs +++ b/src/GameCult.Networking/CultNetShardMutationLogStorage.cs @@ -10,6 +10,11 @@ namespace GameCult.Networking /// /// Stores accepted shard-log entries in their replica catch-up wire shape. /// + /// + /// A database calls its store from a cache journal, under that cache's gate, so a slow store slows every writer of + /// the cache and blocks its readers too, because reads take the same gate. A store must not call the cache, or wait for a thread that does. Appends must throw rather than + /// swallow a failure: the database burns the sequence and compacts past it, and tells the writer. + /// public interface ICultNetShardMutationLogStore { /// diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index e98fdc03..9799cb38 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -768,9 +768,11 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence. + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random: + /// every session is a new identity to its peer, and an options object reused for several + /// must not give them one sequence. /// - public uint InitialSequence { get; set; } = 1; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -845,9 +847,11 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence. + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random: + /// every session is a new identity to its peer, and an options object reused for several + /// must not give them one sequence. /// - public uint InitialSequence { get; set; } = 1; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -882,18 +886,17 @@ namespace GameCult.Networking /// Maximum reliable packets admitted to the wire before acknowledgements advance the window. public const int ReliableSendWindowPackets = 32; private const int ReceivedSequenceWindow = 4096; + // How long a client keeps retransmitting a Connect nobody answered before it abandons that + // attempt for a fresh one. A Connect the server answers with an Ack, never an Accept, is one + // the server judged stale (see AcceptConnect); retransmitting the same sequence would never + // change its mind. + private const long ConnectAttemptMs = 3_000; private sealed class PendingReliablePacket { public CultNetRudpPacket Packet { get; set; } = new CultNetRudpPacket(); public long LastSentAtMs { get; set; } } - private sealed class PendingOrderedFrame - { - public CultNetRudpDeliveredFrame Frame { get; set; } = new CultNetRudpDeliveredFrame(); - public uint NextSequence { get; set; } - } - private sealed class FragmentBuffer { public string ChannelId { get; set; } = string.Empty; @@ -908,6 +911,24 @@ namespace GameCult.Networking private ushort _nextFragmentId = 1; private readonly int? _maxPendingReliablePackets; private volatile bool _connected; + // Advances every time a generation ends. Everything issued in a generation (writes, flushes) + // belongs to it and dies with it. + private long _generation; + // True from the end of a generation until the next Connect or Accept begins one. A flush started + // in that interval has no live generation to wait on. + private bool _ended; + // The sequence of the Connect that started the current generation: sent by this side, or + // accepted from the peer. A Connect repeats exactly when the session is connected and the + // Connect carries this sequence. + private uint? _connectSequence; + // A Connect this side sent is unanswered. Only then is an Accept honoured. + private bool _awaitingAccept; + // When the unanswered Connect first went out; DueResends abandons it for a fresh attempt once + // ConnectAttemptMs has passed. + private long _connectStartedAtMs; + // The payload of the Connect this side sent, kept so a fresh attempt can always be built + // whatever became of the pending packet. + private byte[] _connectPayload = Array.Empty(); private long? _lastReceivedAtMs; private uint? _highestReceivedSequence; private readonly HashSet _receivedSequences = new HashSet(); @@ -915,9 +936,14 @@ namespace GameCult.Networking private readonly object _pendingReliableGate = new object(); private readonly Dictionary _pendingReliable = new Dictionary(); private readonly Queue _queuedReliable = new Queue(); - private readonly Dictionary _orderedNextSequenceByChannel = new Dictionary(StringComparer.Ordinal); - private readonly Dictionary> _orderedBuffers = - new Dictionary>(StringComparer.Ordinal); + // Every reliable sequence up to and including this one has been received since the peer state + // was last reset. Only the handshake seeds it: the peer's Connect on the accepting side, the + // peer's Accept on the connecting side. Reliable data that arrives before that is refused. + private uint? _receivedThrough; + // Ordered frames received but not yet deliverable, keyed by first sequence. A frame is held for + // exactly one reason: a reliable sequence below it has not arrived. + private readonly SortedDictionary _orderedHeld = + new SortedDictionary(); private readonly Dictionary _fragmentBuffers = new Dictionary(StringComparer.Ordinal); /// @@ -929,18 +955,39 @@ namespace GameCult.Networking ValidateLimits(options.InitialSequence, options.MaxPendingReliablePackets); ConnectionId = options.ConnectionId; - _nextSequence = options.InitialSequence; + _nextSequence = options.InitialSequence ?? DrawSequence(); ResendDelayMs = options.ResendDelayMs; _maxPendingReliablePackets = options.MaxPendingReliablePackets; } + /// + /// A sequence drawn from a secure source in [1, 2^31). The Connect's sequence is what tells a + /// peer whether a Connect repeats one it already accepted or starts a new session, so two + /// sessions must not share one by default. + /// + private static uint DrawSequence() + { + const uint ceiling = 1u << 31; + var bytes = new byte[4]; + using (var random = System.Security.Cryptography.RandomNumberGenerator.Create()) + random.GetBytes(bytes); + return 1u + BitConverter.ToUInt32(bytes, 0) % (ceiling - 1u); + } + + // Whether `sequence` is at or before `mark` in serial order, within the receive window. The + // compare is modular, so it holds across the wrap of the 32-bit space; a sequence further back + // than the window is the duplicate test's below-window clause, and one ahead of the mark is + // never before it. + private static bool AtOrBefore(uint sequence, uint mark) => + unchecked(mark - sequence) < ReceivedSequenceWindow; + internal static readonly byte[] RefusedPacketReason = Encoding.UTF8.GetBytes("session refused a packet"); /// /// A session cannot admit a Connect when its sequence space starts exhausted or its reliable /// queue holds nothing, so neither is a usable configuration. /// - internal static void ValidateLimits(uint initialSequence, int? maxPendingReliablePackets) + internal static void ValidateLimits(uint? initialSequence, int? maxPendingReliablePackets) { if (initialSequence == uint.MaxValue) { @@ -965,6 +1012,8 @@ namespace GameCult.Networking /// Gets whether the session has completed the connect/accept handshake. /// public bool Connected => _connected; + internal long Generation => Interlocked.Read(ref _generation); + internal bool Ended => _ended; /// /// Gets the logical time of the last received packet. /// @@ -1000,14 +1049,57 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateConnect(long nowMs = 0, byte[]? payload = null) { + // A session that has had a peer starts a new generation: nothing it learned from that + // peer describes the one this Connect reaches, and what it still owed that peer no + // longer takes room in the queue. + if (_connectSequence.HasValue) + ResetPeerState(); EnsureReliableCapacity(1); + _ended = false; var packet = CreatePacket(CultNetRudpPacketType.Connect, "control", payload ?? Array.Empty(), reliable: true, ordered: true, sequenced: false); + _connectSequence = packet.Sequence; + _awaitingAccept = true; + _connectStartedAtMs = nowMs; + _connectPayload = packet.Payload ?? Array.Empty(); TrackReliable(packet, nowMs); return packet; } /// - /// Accepts a connect packet and returns a reliable ordered accept packet. + /// Whether is a Connect this session's current generation already owns: + /// a retransmit of the Connect that started it (the session is connected and the sequence is + /// that Connect's), or a stale copy of an earlier attempt by the same client (a sequence before + /// it, within the receive window). Servers that keep one session per peer ask this to tell a + /// Connect that starts a new session from one that does not; anything else that reaches + /// starts a new generation. + /// + public bool ConnectRepeats(CultNetRudpPacket packet) + { + if (packet == null) throw new ArgumentNullException(nameof(packet)); + return packet.PacketType == CultNetRudpPacketType.Connect + && _connected + && _connectSequence.HasValue + && (_connectSequence.Value == packet.Sequence + || ConnectIsStale(packet.Sequence, _connectSequence.Value)); + } + + // A Connect that precedes the current generation's within the receive window is the client's + // earlier attempt, delayed in the network: a client that retried never sends a lower sequence + // again. Restarting on it would strand the client, which honours only the Accept for its newest + // Connect. The rule is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): keep the + // connection and answer with an Ack. A restarted client whose random initial sequence lands in + // this window is answered the same way and abandons the attempt for a fresh draw + // (ConnectAttemptMs). + private static bool ConnectIsStale(uint sequence, uint current) => + sequence != current && AtOrBefore(sequence, current); + + /// + /// Answers a connect packet. A repeat of the accepted connect queues nothing, so a connect storm + /// cannot grow the reliable queue: the reply is the accept still awaiting acknowledgement, or an + /// ack once it was acknowledged. A stale copy of an earlier attempt gets the same reply and + /// changes nothing else: it is not evidence the peer is alive. Any other connect ends the + /// current generation, forgets the peer and accepts a new one with a reliable ordered accept + /// packet. /// public CultNetRudpPacket AcceptConnect(CultNetRudpPacket packet, long nowMs = 0, byte[]? payload = null) { @@ -1017,30 +1109,31 @@ namespace GameCult.Networking throw new InvalidOperationException($"Expected RUDP connect packet, got {packet.PacketType}."); } + if (ConnectRepeats(packet)) + { + if (_connectSequence == packet.Sequence) + { + ApplyAcknowledgements(packet); + RememberReceived(packet.Sequence); + _lastReceivedAtMs = nowMs; + } + return PendingAcceptForResend(nowMs) ?? CreateAck(); + } + ResetPeerState(); EnsureReliableCapacity(1); - RememberReceived(packet.Sequence); + SeedReceived(packet.Sequence); + _lastReceivedAtMs = nowMs; + _connectSequence = packet.Sequence; _connected = true; + _ended = false; var response = CreatePacket(CultNetRudpPacketType.Accept, "control", payload ?? Array.Empty(), reliable: true, ordered: true, sequenced: false); TrackReliable(response, nowMs); return response; } - /// - /// Answers a connect from a peer this session has already accepted. The packet's sequence is - /// remembered and nothing is queued, so a connect storm cannot grow the reliable queue. The reply - /// is the accept still awaiting acknowledgement, or an ack once it was acknowledged. - /// - public CultNetRudpPacket AnswerRepeatedConnect(CultNetRudpPacket packet, long nowMs = 0) + // The Accept still awaiting acknowledgement, resent. + private CultNetRudpPacket? PendingAcceptForResend(long nowMs) { - RequireConnection(packet); - if (packet.PacketType != CultNetRudpPacketType.Connect) - { - throw new InvalidOperationException($"Expected RUDP connect packet, got {packet.PacketType}."); - } - - ApplyAcknowledgements(packet); - RememberReceived(packet.Sequence); - _lastReceivedAtMs = nowMs; lock (_pendingReliableGate) { foreach (var pending in _pendingReliable.Values) @@ -1052,7 +1145,7 @@ namespace GameCult.Networking } } - return CreateAck(); + return null; } /// @@ -1145,16 +1238,29 @@ namespace GameCult.Networking public CultNetRudpReceiveResult Receive(CultNetRudpPacket packet, long nowMs = 0) { RequireConnection(packet); - ApplyAcknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the Accept names it. + // A late or duplicate one, or one from an earlier generation, must not seed the watermark + // or revive an ended session. + var honoursAccept = packet.PacketType == CultNetRudpPacketType.Accept + && _awaitingAccept + && _connectSequence.HasValue + && PacketAcknowledges(packet, _connectSequence.Value); + if (packet.PacketType == CultNetRudpPacketType.Accept && !honoursAccept) + { + return new CultNetRudpReceiveResult(); + } + // While this side's Connect awaits its Accept, only the Accept it honours retires it: an Ack + // that names the Connect (a server's reply to a repeat or a stale copy) says the server did + // not start a session. + if (honoursAccept || !_awaitingAccept) + ApplyAcknowledgements(packet); var readyToSend = PromoteQueuedReliable(nowMs); _lastReceivedAtMs = nowMs; - var expectedSequenceIfUninitialized = _highestReceivedSequence.HasValue - ? _highestReceivedSequence.Value + 1 - : packet.Sequence; - if (packet.PacketType == CultNetRudpPacketType.Accept) + if (honoursAccept) { - RememberReceived(packet.Sequence); + _awaitingAccept = false; + SeedReceived(packet.Sequence); _connected = true; return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } @@ -1189,7 +1295,7 @@ namespace GameCult.Networking if (packet.PacketType == CultNetRudpPacketType.Disconnect) { - _connected = false; + EndSession(); return new CultNetRudpReceiveResult { ReadyToSend = readyToSend, @@ -1203,11 +1309,15 @@ namespace GameCult.Networking return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } - var duplicate = packet.Reliable - && (_receivedSequences.Contains(packet.Sequence) - || (_highestReceivedSequence.HasValue - && packet.Sequence < _highestReceivedSequence.Value - && _highestReceivedSequence.Value - packet.Sequence >= ReceivedSequenceWindow)); + // Reliable data before the handshake has seeded the watermark has no place in the order: + // refuse it unremembered, so it is not acknowledged and the sender retransmits it once + // the handshake is done. + if (packet.Reliable && !_receivedThrough.HasValue) + { + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + } + + var duplicate = packet.Reliable && WasReceived(packet.Sequence); if (packet.Reliable) { RememberReceived(packet.Sequence); @@ -1217,30 +1327,34 @@ namespace GameCult.Networking return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } + var delivered = new List(); var reassembled = Reassemble(packet); - if (reassembled == null) + if (reassembled != null) { - return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; - } - - if (!reassembled.Ordered && packet.Sequenced) - { - var newestSequence = reassembled.NextSequence - 1; - if (_latestSequencedByChannel.TryGetValue(reassembled.Frame.ChannelId, out var latestSequence) - && newestSequence <= latestSequence) + if (reassembled.Ordered) { - return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + _orderedHeld[reassembled.Frame.Sequence] = reassembled.Frame; + } + else if (!packet.Sequenced) + { + delivered.Add(reassembled.Frame); + } + else + { + var newestSequence = reassembled.NextSequence - 1; + if (!_latestSequencedByChannel.TryGetValue(reassembled.Frame.ChannelId, out var latestSequence) + || newestSequence > latestSequence) + { + _latestSequencedByChannel[reassembled.Frame.ChannelId] = newestSequence; + delivered.Add(reassembled.Frame); + } } - _latestSequencedByChannel[reassembled.Frame.ChannelId] = newestSequence; } - return new CultNetRudpReceiveResult - { - ReadyToSend = readyToSend, - Delivered = reassembled.Ordered - ? DeliverOrdered(reassembled.Frame, reassembled.NextSequence, expectedSequenceIfUninitialized) - : new[] { reassembled.Frame } - }; + // Any reliable packet may have advanced the watermark, a fragment or an unordered frame as + // much as an ordered one, so the drain runs after all of them. + delivered.AddRange(DrainOrdered()); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend, Delivered = delivered }; } /// @@ -1272,6 +1386,10 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateAckForReceived(uint receivedSequence) { + // What Receive refused is not acknowledged by name (the ack carries only what was + // received), so its sender retransmits it. + if (!WasReceived(receivedSequence)) + return CreateAck(); var (ack, _) = AckState(); return ack >= receivedSequence && ack - receivedSequence <= 32 ? CreateAck() @@ -1287,23 +1405,39 @@ namespace GameCult.Networking } /// - /// Forgets everything learned from the peer; sequence numbers already issued stay issued. + /// The one way a session generation ends. The session stops being connected and what it still + /// owed the peer dies with it: a write not yet acknowledged is dropped, so no later session + /// retransmits it or credits an ack to it. What was learned from the peer is not touched: the + /// peer may not know the session ended, and forgetting what it sent would let its retransmits + /// be delivered twice. /// - public void ResetPeerState() + private void EndSession() { _connected = false; - _lastReceivedAtMs = null; - _highestReceivedSequence = null; - _receivedSequences.Clear(); - _nextSequencedByChannel.Clear(); - _latestSequencedByChannel.Clear(); + _ended = true; + _awaitingAccept = false; + Interlocked.Increment(ref _generation); lock (_pendingReliableGate) { _pendingReliable.Clear(); _queuedReliable.Clear(); } - _orderedNextSequenceByChannel.Clear(); - _orderedBuffers.Clear(); + } + + /// + /// Ends the current generation and forgets everything learned from the peer; sequence numbers + /// already issued stay issued. + /// + public void ResetPeerState() + { + EndSession(); + _lastReceivedAtMs = null; + _highestReceivedSequence = null; + _receivedSequences.Clear(); + _nextSequencedByChannel.Clear(); + _latestSequencedByChannel.Clear(); + _receivedThrough = null; + _orderedHeld.Clear(); _fragmentBuffers.Clear(); } @@ -1323,7 +1457,7 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateDisconnect(byte[]? reason = null) { - _connected = false; + EndSession(); return CreatePacket(CultNetRudpPacketType.Disconnect, "control", reason ?? Array.Empty(), reliable: false, ordered: false, sequenced: false); } @@ -1340,7 +1474,7 @@ namespace GameCult.Networking { return false; } - _connected = false; + EndSession(); return true; } @@ -1349,6 +1483,9 @@ namespace GameCult.Networking /// public IReadOnlyList DueResends(long nowMs) { + var fresh = AbandonUnansweredConnect(nowMs); + if (fresh != null) + return new[] { fresh }; var due = new List(); lock (_pendingReliableGate) { @@ -1515,9 +1652,69 @@ namespace GameCult.Networking } } + private static bool PacketAcknowledges(CultNetRudpPacket packet, uint sequence) + { + if (packet.Ack == sequence) + return true; + for (var bit = 0; bit < 32; bit++) + { + if ((packet.AckMask & (1u << bit)) != 0 && packet.Ack > bit && packet.Ack - (uint)bit - 1 == sequence) + return true; + } + return false; + } + + // A Connect unanswered for ConnectAttemptMs is replaced, not retransmitted further, by a Connect + // with the same payload and the abandoned sequence plus the receive window less one. Nothing + // was ever sent in an unanswered generation, so no sequence issued so far is owed. The jump + // keeps a late copy of the abandoned Connect inside the new one's stale window, so a server + // that took the new Connect answers the copy with an Ack instead of restarting; and it leaves + // the stale window of whatever generation the server holds, unless that generation sits + // exactly at the jump, and then the next attempt leaves it. The first Connect of a session is + // the only one drawn at random. + private CultNetRudpPacket? AbandonUnansweredConnect(long nowMs) + { + if (!_awaitingAccept || nowMs - _connectStartedAtMs < ConnectAttemptMs || !_connectSequence.HasValue) + return null; + var jumped = (ulong)_connectSequence.Value + ReceivedSequenceWindow - 1; + _nextSequence = (uint)Math.Min(jumped, uint.MaxValue - 1); + return CreateConnect(nowMs, _connectPayload); + } + + // The handshake's one act on the watermark: the peer's Connect or Accept is the first sequence + // of the session, whatever else it has sent. + private void SeedReceived(uint sequence) + { + _receivedThrough = sequence; + RememberReceived(sequence); + } + + // The one duplicate test: true for a sequence in the window that was received, for one below + // the window, and for one at or before the watermark. The watermark starts at the handshake's + // seed, so a frame the peer sent in an earlier generation (every sequence it issued is below + // the Connect that began this one) is a duplicate of something already delivered and is + // acknowledged, never delivered again. + private bool WasReceived(uint sequence) + { + return _receivedSequences.Contains(sequence) + || (_highestReceivedSequence.HasValue + && sequence < _highestReceivedSequence.Value + && _highestReceivedSequence.Value - sequence >= ReceivedSequenceWindow) + || (_receivedThrough.HasValue && AtOrBefore(sequence, _receivedThrough.Value)); + } + private void RememberReceived(uint sequence) { _receivedSequences.Add(sequence); + if (_receivedThrough.HasValue) + { + var through = _receivedThrough.Value; + while (through != uint.MaxValue && _receivedSequences.Contains(through + 1)) + { + through++; + } + _receivedThrough = through; + } if (!_highestReceivedSequence.HasValue || sequence > _highestReceivedSequence.Value) { _highestReceivedSequence = sequence; @@ -1625,80 +1822,35 @@ namespace GameCult.Networking }; } - private IReadOnlyList DeliverOrdered( - CultNetRudpDeliveredFrame frame, - uint nextSequenceAfterFrame, - uint expectedSequenceIfUninitialized) - { - if (!_orderedNextSequenceByChannel.TryGetValue(frame.ChannelId, out var next)) - { - next = Math.Min(expectedSequenceIfUninitialized, frame.Sequence); - _orderedNextSequenceByChannel[frame.ChannelId] = next; - } - - while (frame.Sequence > next - && _receivedSequences.Contains(next) - && (!_orderedBuffers.TryGetValue(frame.ChannelId, out var pendingBuffer) || !pendingBuffer.ContainsKey(next))) - { - next++; - _orderedNextSequenceByChannel[frame.ChannelId] = next; - } - - if (frame.Sequence < next) - { - return Array.Empty(); - } - - if (frame.Sequence > next) - { - if (!_orderedBuffers.TryGetValue(frame.ChannelId, out var buffer)) - { - buffer = new SortedDictionary(); - _orderedBuffers[frame.ChannelId] = buffer; - } - - buffer[frame.Sequence] = new PendingOrderedFrame { Frame = frame, NextSequence = nextSequenceAfterFrame }; - return Array.Empty(); - } - - _orderedNextSequenceByChannel[frame.ChannelId] = nextSequenceAfterFrame; - return new[] { frame }.Concat(DrainOrdered(frame.ChannelId)).ToArray(); - } - - private IReadOnlyList DrainOrdered(string channelId) + /// + /// Delivers, in sequence order, every held ordered frame whose first sequence is at most one past + /// the watermark. Ordered delivery has one owner: the watermark. A frame this call does not + /// deliver waits for the sequence below it, and nothing else releases it. + /// + private IReadOnlyList DrainOrdered() { var delivered = new List(); - if (!_orderedBuffers.TryGetValue(channelId, out var buffer)) + if (!_receivedThrough.HasValue) { return delivered; } - while (_orderedNextSequenceByChannel.TryGetValue(channelId, out var next) && buffer.TryGetValue(next, out var pending)) + var through = _receivedThrough.Value; + foreach (var pair in _orderedHeld) { - buffer.Remove(next); - delivered.Add(pending.Frame); - _orderedNextSequenceByChannel[channelId] = pending.NextSequence; - SkipReceivedNonChannelSequences(channelId); + if (through != uint.MaxValue && pair.Key > through + 1) + { + break; + } + delivered.Add(pair.Value); + } + foreach (var frame in delivered) + { + _orderedHeld.Remove(frame.Sequence); } - return delivered; } - private void SkipReceivedNonChannelSequences(string channelId) - { - if (!_orderedNextSequenceByChannel.TryGetValue(channelId, out var next)) - { - return; - } - - while (_receivedSequences.Contains(next) - && (!_orderedBuffers.TryGetValue(channelId, out var buffer) || !buffer.ContainsKey(next))) - { - next++; - _orderedNextSequenceByChannel[channelId] = next; - } - } - private ushort AllocateFragmentId() { var fragmentId = _nextFragmentId; @@ -1917,18 +2069,23 @@ namespace GameCult.Networking var preserved = new List(); while (_deliveredFrames.Count > 0) preserved.Add(_deliveredFrames.Dequeue()); + // A flush belongs to the generation it started in: whatever ends that generation, and + // whatever begins after it, the writes being waited on are gone. + long generation; + lock (_sessionGate) + generation = _session.Generation; try { while (true) { - // An ended session forgot its unacknowledged writes; reporting them - // flushed would be a lie. - var ended = DisconnectReason; - if (ended != null) - throw new InvalidOperationException( - "RUDP session ended before its reliable writes were acknowledged: " + Encoding.UTF8.GetString(ended)); lock (_sessionGate) { + // An ended session forgot its unacknowledged writes; reporting them flushed + // would be a lie. + if (_session.Ended || _session.Generation != generation) + throw new InvalidOperationException( + "RUDP session ended before its reliable writes were acknowledged: " + + (DisconnectReason == null ? "no reason given" : Encoding.UTF8.GetString(DisconnectReason))); if (_session.OutstandingReliablePacketCount == 0) return; } @@ -2171,7 +2328,6 @@ namespace GameCult.Networking error.SocketErrorCode == SocketError.ConnectionAborted || error.SocketErrorCode == SocketError.Shutdown) { - DisconnectReason = Array.Empty(); return false; } @@ -2198,6 +2354,10 @@ namespace GameCult.Networking _stats.PacketsDropped++; return true; } + // A Connect from another endpoint is a new client, whatever sequence it carries: the endpoint + // moves and the Connect starts a new generation. Only a Connect from the peer's own + // endpoint can be its repeat. + var connectFromNewEndpoint = false; if (_remoteEndPoint == null) { // Only a Connect claims the endpoint of a server-mode transport. @@ -2211,8 +2371,16 @@ namespace GameCult.Networking } else if (!_remoteEndPoint.Equals(remote)) { - _stats.PacketsDropped++; - return true; + if (_mode == CultNetRudpSocketMode.Server && packet.PacketType == CultNetRudpPacketType.Connect) + { + _remoteEndPoint = remote; + connectFromNewEndpoint = true; + } + else + { + _stats.PacketsDropped++; + return true; + } } TracePacket("rx", packet, received, remote); @@ -2223,17 +2391,12 @@ namespace GameCult.Networking CultNetRudpPacket accept; try { - // A connect from the peer this transport already accepted repeats: answer it - // with the accept already owed and queue nothing. - if (_session.Connected) - { - accept = _session.AnswerRepeatedConnect(packet, NowMs()); - } - else - { - accept = _session.AcceptConnect(packet, NowMs()); - DisconnectReason = null; - } + // The session answers a repeat of the connect it accepted with the accept + // already owed, and anything else with a new generation. + if (connectFromNewEndpoint) + _session.ResetPeerState(); + accept = _session.AcceptConnect(packet, NowMs()); + DisconnectReason = null; } catch (InvalidOperationException) { @@ -2399,9 +2562,10 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence for each accepted peer session. + /// Gets or sets the first local packet sequence for each accepted peer session. Unset, each + /// session draws its own at random. /// - public uint InitialSequence { get; set; } = 100; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -2493,7 +2657,7 @@ namespace GameCult.Networking private const int WireBurstPackets = 32; private readonly Socket _socket; private readonly uint _connectionId; - private readonly uint _initialSequence; + private readonly uint? _initialSequence; private readonly long _resendDelayMs; private readonly int? _maxFragmentBytes; private readonly int? _maxPendingReliablePackets; @@ -2668,25 +2832,29 @@ namespace GameCult.Networking { var connectPayload = packet.Payload ?? Array.Empty(); _peers.TryGetValue(peerKey, out var admitted); - // The same payload from an admitted peer is a repeated Connect: answer it with the - // Accept already owed and queue nothing. A different payload is a new generation. - if (admitted != null && admitted.ConnectPayload.AsSpan().SequenceEqual(connectPayload)) + // The admitted peer's session decides whether this Connect repeats the one it accepted: + // answer a repeat with the Accept already owed and queue nothing. Any other Connect is + // a new generation. + if (admitted != null) { lock (admitted.SessionGate) { - CultNetRudpPacket reply; - try + if (admitted.Session.ConnectRepeats(packet)) { - reply = admitted.Session.AnswerRepeatedConnect(packet, NowMs()); - } - catch (InvalidOperationException) - { - _stats.PacketsDropped++; + CultNetRudpPacket reply; + try + { + reply = admitted.Session.AcceptConnect(packet, NowMs(), _acceptPayload); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } + SendPacket(admitted.RemoteEndPoint, reply); return true; } - SendPacket(admitted.RemoteEndPoint, reply); } - return true; } var peer = new CultNetRudpSocketServerPeer( diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index 5c7f2a23..90e7b1e7 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -178,6 +178,61 @@ namespace GameCult.Caching.Tests Throws.TypeOf().With.Message.Contains("format version")); } + // Every runtime reads a store file the same way: a missing file is an empty store, an existing empty file is not + // a store (no CultCache writer leaves one), and an empty array carries no header, so it claims no format and is an + // empty store too. + [Test] + public async Task SingleFileRefusesAnEmptyFileAndOpensAMissingFileOrAnEmptyArrayAsEmpty() + { + var root = Path.Combine(Path.GetTempPath(), $"cultlib-empty-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + var empty = Path.Combine(root, "empty.msgpack"); + File.WriteAllBytes(empty, Array.Empty()); + using (var cache = new CultCache()) + { + Assert.Catch(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(empty))); + Assert.That(cache.BackingStores, Is.Empty); + } + Assert.That(File.ReadAllBytes(empty), Is.Empty); + + var emptyArray = Path.Combine(root, "empty-array.msgpack"); + File.WriteAllBytes(emptyArray, new byte[] { 0x90 }); + foreach (var path in new[] { Path.Combine(root, "missing.msgpack"), emptyArray }) + { + using var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.PullAllBackingStoresAsync(); + Assert.That(cache.GetAll(), Is.Empty, path); + } + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + // A refusal echoes a header only in the shape cultcache.store.v; any other is described by its length. + [TestCase("cultcache.store.v12", true)] + [TestCase("cultcache.store.SECRET-HEADER", false)] + [TestCase("cultcache.store.v12SECRET", false)] + [TestCase("cultcache.store.v", false)] + public void SingleFileRefusalEchoesAHeaderOnlyInTheKnownShape(string header, bool echoed) + { + var snapshot = new CultPersistedStoreSnapshot { FormatVersion = header }; + var error = Assert.Throws(() => CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot))!; + if (echoed) + { + Assert.That(error.Message, Does.Contain("format " + header + " is not")); + } + else + { + Assert.That(error.Message, Does.Not.Contain("SECRET").And.Not.Contain("format " + header + " is not")); + Assert.That(error.Message, Does.Contain($"of {header.Length} bytes")); + } + } + [Test] public void V1StoreWrittenAtTheBaseCommitStillDecodesByteForByte() { diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs index 4f0808bf..31b5f325 100644 --- a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -426,7 +426,6 @@ namespace GameCult.Caching.Tests // A commit that lands onto the file as it is keeps the header the file carries: a store already v3 stays v3. [TestCase("../v3-base.msgpack", "cultcache.store.v3")] [TestCase("compatible-id-only-v3.bin", "cultcache.store.v3")] - [TestCase("zero-byte.bin", "cultcache.store.v1")] [TestCase("empty-array.bin", "cultcache.store.v1")] public void AConditionalCommitOntoAReadableFileKeepsItsHeader(string vector, string header) { diff --git a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs index 1eb1573d..b960294b 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs @@ -902,6 +902,32 @@ public sealed class CultMeshStreamingTests await act.Should().ThrowAsync(); } + // A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type + // out of the cache behind it. + [TestCase("tests.mesh_note.v1", true)] + [TestCase("tests.unrelated.v1", false)] + public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served) + { + var cache = new CultCache(); + var key = new CultRecordKey("note:one"); + await cache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cached" }, new CultRecordHandle(key)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "reader", + Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) } + }); + var handle = CultMesh.Document(database, key, CultMesh.Verse("starbridge", "reader")); + + if (served) + { + (await handle.LatestAsync()).Text.Should().Be("cached"); + } + else + { + await FluentActions.Awaiting(() => handle.LatestAsync()).Should().ThrowAsync(); + } + } + [Test] public async Task DocumentHandle_SubmitsPredictionsThroughCultNetDatabase() { diff --git a/tests/GameCult.Networking.InteropPeer/Program.cs b/tests/GameCult.Networking.InteropPeer/Program.cs index f07908a9..17a1ea89 100644 --- a/tests/GameCult.Networking.InteropPeer/Program.cs +++ b/tests/GameCult.Networking.InteropPeer/Program.cs @@ -873,6 +873,7 @@ static void RudpServeOnce(Dictionary options) } if (disconnectReason != null) { + transport.FlushReliable(TimeSpan.FromSeconds(2)); transport.Disconnect(disconnectReason); } PollRudpAfterSend(transport, TimeSpan.FromMilliseconds(250)); diff --git a/tests/GameCult.Networking.Tests/CultNetJournalTests.cs b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs new file mode 100644 index 00000000..6285d674 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetJournalTests.cs @@ -0,0 +1,1199 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // The database's shard log is written by a cache journal, under the cache gate, in cache order. A primary that + // cannot log a committed change burns that change's sequence and compacts past it, so the log can lag the cache but + // never lead it, and every replica behind the hole resynchronizes by snapshot. + public sealed class CultNetJournalTests + { + private const string ShardId = "log-shard"; + + private static readonly CultRecordKey One = new("tests:journal:one"); + private static readonly CultRecordKey Two = new("tests:journal:two"); + private static readonly CultRecordKey Three = new("tests:journal:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetShardDescriptor Shard(CultCache cache, bool primary) => + new(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)]); + + private static CultNetDatabase Database(CultCache cache, bool primary, FlakyLogStore? store) => + new(cache, new CultNetDatabaseOptions { Shards = [Shard(cache, primary)], MutationLogStore = store }); + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + private static List Record(CultNetDatabase database) + { + var seen = new List(); + database.WatchAllChanges().Subscribe(change => seen.Add(KeyOfChange(change))); + return seen; + } + + private static long[] Sequences(FlakyLogStore store) => store.Read(ShardId).Select(entry => entry.Sequence).ToArray(); + + // A store whose refusals and side effects the test scripts. One shard per store. + private sealed class FlakyLogStore : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compactedThrough; + + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public Action? OnRead { get; set; } + public Action? OnAppend { get; set; } + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + OnRead?.Invoke(); + lock (_gate) + { + var entries = _entries.Values.Where(entry => entry.Sequence > afterSequence); + return (limit.HasValue ? entries.Take(limit.Value) : entries).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + OnAppend?.Invoke(); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) + throw new IOException($"The log refused sequence {entry.Sequence}."); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) + { + lock (_gate) + return _compactedThrough; + } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) + throw new IOException("The log refused to compact."); + _compactedThrough = Math.Max(_compactedThrough, sequence); + foreach (var stale in _entries.Keys.Where(key => key <= sequence).ToArray()) + _entries.Remove(stale); + } + } + } + + // The primary as a replica meets it: the database server's own log answer, and the database's own snapshot. + private sealed class PrimaryPeer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _databaseServer; + private readonly CultNetDatabase _database; + + public PrimaryPeer(CultCache cache, CultNetDatabase database) + { + _database = database; + _server = new Server(cache, ServerSecurityOptions.Development()); + _databaseServer = new CultNetDatabaseServer(_server, database); + } + + public CultNetShardLogResponseMessage Log(long afterSequence) => + _databaseServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { + MessageId = "pull", + ShardId = ShardId, + ShardEpoch = 1, + AfterSequence = afterSequence + }); + + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => + Task.FromResult(Log(afterSequence)); + + public Task FetchAsync(CultNetShardDescriptor shard) => + Task.FromResult(_database.CreateShardSnapshotResponse(_database.Shards[0], "snapshot")); + + public void Dispose() + { + _databaseServer.Dispose(); + _server.Dispose(); + } + } + + private static CultNetShardReplicator Replicator(CultNetDatabase replica, PrimaryPeer peer) => + new(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + + private static string? TextOf(CultCache cache, CultRecordKey key) => cache.Get(key)?.Text; + + [Test] + public async Task ARefusedPrimaryAppendPublishesTheChangeTheWriterHearsOfItAndAReplicaBehindItResyncs() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(1)); + var published = Record(primary); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 2) })); + Assert.That(failure, Is.Not.InstanceOf(), "a committed write must not read as a refused one"); + Assert.That(published, Is.EqualTo(new[] { Two.Value }), "the change was published before the writer heard"); + Assert.That(TextOf(cache, Two), Is.EqualTo("two"), "the commit stands"); + Assert.That(primary.LastWriteSequence(SchemaId(cache), Two), Is.EqualTo(2L), "the row was written at the burned sequence"); + await primary.PutAsync(Three, Note("three")); + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "sequence 2 stays a hole below the compaction point"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(3), "a later pull is a quiet no-op"); + } + + [Test] + public async Task ARestartAfterARefusedTailAppendReusesNoSequence() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using (var before = new PrimaryPeer(cache, primary)) + { + await primary.PutAsync(One, Note("one")); + using var replicator = Replicator(replica, before); + await replicator.PullOnceAsync(ShardId); + } + + Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two"))); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L }), "the burned sequence was not minted again"); + using var after = new PrimaryPeer(cache, restarted); + using var resumed = Replicator(replica, after); + await resumed.PullOnceAsync(ShardId); + Assert.That(new[] { One, Two, Three }.Select(key => TextOf(replicaCache, key)), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test] + public async Task AStoreThatRefusesTheAppendAndTheCompactionStillCompactsInMemoryAndNamesBothFailures() + { + var store = new FlakyLogStore { RefuseAppend = sequence => sequence == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + + var failure = Assert.ThrowsAsync(async () => await primary.PutAsync(Two, Note("two")))!; + + Assert.That(failure.InnerException, Is.InstanceOf()); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2)); + Assert.That(primary.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2), "the primary's own floor covers a hole its store never compacted"); + Assert.That(primary.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the floor"); + Assert.That(peer.Log(afterSequence: 1).Reason, Is.EqualTo("compacted_history")); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two")); + } + + [Test] + public async Task CompactingAndRestartingResumesTheNextSequenceAboveTheCompactionPoint() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + store.CompactThrough(ShardId, 2); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "the latest sequence is never below the compaction point"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 3L })); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(3)); + } + + [Test] + public async Task ARefusedReplicaAppendRecordsNothingAdvancesNoCursorAndTheRetryConverges() + { + var refusals = 0; + var replicaStore = new FlakyLogStore { RefuseAppend = sequence => sequence == 1 && refusals++ == 0 }; + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + using var replicator = Replicator(replica, peer); + + Assert.ThrowsAsync(async () => await replicator.PullOnceAsync(ShardId)); + + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(0)); + Assert.That(replica.GetMutationLog(ShardId), Is.Empty, "memory never holds what the durable log refused"); + Assert.That(await replicator.PullOnceAsync(ShardId), Is.EqualTo(2)); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L })); + } + + [Test] + public async Task AReentrantShardLogApplyFromAHandlerKeepsTheReplicaLogInPrimaryOrder() + { + var cache = new CultCache(); + var primary = Database(cache, primary: true, new FlakyLogStore()); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + await primary.PutAsync(Three, Note("three")); + var response = new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = primary.GetMutationLogMessages(ShardId).ToArray() + }; + var replicaStore = new FlakyLogStore(); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, replicaStore); + var nested = false; + long nestedApplied = 0; + Exception? nestedFailure = null; + replica.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + { + if (nested) return; + nested = true; + try + { + nestedApplied = replica.ApplyShardLogResponseAsync(response).GetAwaiter().GetResult(); + } + catch (Exception exception) + { + nestedFailure = exception; + } + }); + + await replica.ApplyShardLogResponseAsync(response); + + Assert.That(nested, Is.True); + Assert.That(nestedFailure, Is.Null, "the nested apply re-records entries the outer apply already logged"); + Assert.That(nestedApplied, Is.EqualTo(3)); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(Sequences(replicaStore), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + Assert.That(replica.CurrentAsOf(ShardId), Is.EqualTo(3UL), "the replica's own watermark follows the primary's sequences"); + } + + [Test] + public async Task AReentrantDatabaseWriteFromAHandlerIsLoggedWithItsContextAndPublishedOnce() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var message = database.Documents.CreateRawDocumentPutMessage( + "reentrant-two", new CultRecordHandle(Two), Note("two")); + var published = Record(database); + database.Watch().Where(change => change.Key.Equals(One)).Subscribe(_ => + database.ApplyPutAsync(message).GetAwaiter().GetResult()); + + await database.PutAsync(One, Note("one")); + + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(store.Read(ShardId).Single(entry => entry.Sequence == 2).Put!.MessageId, Is.EqualTo("reentrant-two"), + "the nested write's own wire message is what the log stored"); + } + + [Test] + public async Task ALogStoreThatWritesTheCacheFailsThroughTheReentryGuardAndTheAdmissionStands() + { + var cache = new CultCache(); + var store = new FlakyLogStore(); + store.OnAppend = () => + { + store.OnAppend = null; + cache.UpsertAsync(Note("from the store"), new CultRecordHandle(Three)).GetAwaiter().GetResult(); + }; + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.ThrowsAsync(async () => await database.PutAsync(One, Note("one")))!; + + Assert.That(failure.InnerException, Is.InstanceOf(), "the cache's guard refused the store's write"); + Assert.That(TextOf(cache, One), Is.EqualTo("one")); + Assert.That(TextOf(cache, Three), Is.Null); + Assert.That(published, Is.EqualTo(new[] { One.Value })); + } + + [Test] + public void SeveralRefusedAppendsInOneAdmissionAreAllPublishedAndAllReachTheWriter() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var published = Record(database); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence(ShardId, 1), new CultNetBurnedSequence(ShardId, 2) }), "one exception carries every burned sequence"); + Assert.That(((AggregateException)failure.InnerException!).InnerExceptions, Has.Count.EqualTo(2), "and every store failure"); + Assert.That(published, Is.EqualTo(new[] { One.Value, Two.Value })); + Assert.That(database.GetCompactedMutationLogSequence(ShardId), Is.EqualTo(2)); + } + + [Test] + public async Task APullOnAPrimaryIsLoggedSoAReplicaConverges() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-pull-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("seed"), new CultRecordHandle(One)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var primary = Database(cache, primary: true, new FlakyLogStore()); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + + await writer.UpsertAsync(Note("pulled"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await replicator.PullOnceAsync(ShardId); + + Assert.That(primary.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Does.Contain(Two.Value)); + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("pulled")); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task AStoreAttachedAfterTheDatabaseExistsHasItsHydrationLogged() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-journal-attach-{Guid.NewGuid():N}.cc"); + try + { + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("one"), new CultRecordHandle(One)); + await writer.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + writer.FlushAllBackingStores(); + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore()); + + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EquivalentTo(new[] { One.Value, Two.Value })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ASnapshotReadsItsLogSequenceBeforeItsDocumentsSoARacingWriteConverges() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + using var peer = new PrimaryPeer(cache, primary); + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + using var replicator = Replicator(replica, peer); + store.OnRead = () => + { + store.OnRead = null; + primary.PutAsync(Two, Note("two")).GetAwaiter().GetResult(); + }; + + var snapshot = primary.CreateShardSnapshotResponse(primary.Shards[0], "snapshot"); + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], snapshot); + await replicator.PullOnceAsync(ShardId); + + Assert.That(TextOf(replicaCache, Two), Is.EqualTo("two"), "a write racing the snapshot reaches the replica by one road or the other"); + } + + [Test] + public async Task ASnapshotAppliedToAPrimaryShardLogsNothing() + { + var cache = new CultCache(); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(One)); + var database = Database(cache, primary: true, new FlakyLogStore()); + var put = database.Documents.CreateRawDocumentPutMessage("snapshot-two", new CultRecordHandle(Two), Note("two")); + + await database.ApplyShardSnapshotResponseAsync(database.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(TextOf(cache, One), Is.Null); + Assert.That(TextOf(cache, Two), Is.EqualTo("two")); + Assert.That(database.GetMutationLog(ShardId), Is.Empty, "the snapshot's removal and its put are both a resync, not a commit"); + } + + [Test] + public async Task ARestartOverARetainedLogResumesAboveItsHighestSequence() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var primary = Database(cache, primary: true, store); + await primary.PutAsync(One, Note("one")); + await primary.PutAsync(Two, Note("two")); + primary.Dispose(); + + var restarted = Database(cache, primary: true, store); + Assert.That(restarted.GetLatestMutationLogSequence(ShardId), Is.EqualTo(2), "a restarted database reads its latest sequence from the durable log"); + await restarted.PutAsync(Three, Note("three")); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L, 2L, 3L })); + } + + [Test] + public async Task APutAndDeleteDoorLogTheirOwnWireMessagesAndChangeKinds() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var added = database.Documents.CreateRawDocumentPutMessage("put-added", new CultRecordHandle(One), Note("one")); + var updated = database.Documents.CreateRawDocumentPutMessage("put-updated", new CultRecordHandle(One), Note("uno")); + + await database.ApplyPutAsync(added); + await database.ApplyPutAsync(updated); + await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage { MessageId = "delete-one", SchemaId = SchemaId(cache), RecordKey = One.Value }); + + var entries = store.Read(ShardId); + Assert.That(entries.Select(entry => entry.ChangeKind), Is.EqualTo(new[] { "added", "updated", "removed" })); + Assert.That(entries[0].Put!.MessageId, Is.EqualTo("put-added")); + Assert.That(entries[1].Put!.MessageId, Is.EqualTo("put-updated")); + Assert.That(entries[2].Delete!.MessageId, Is.EqualTo("delete-one")); + Assert.That(DateTimeOffset.TryParse(entries[0].CommittedAt, out _), Is.True, "a primary stamps its entries"); + } + + [Test] + public async Task AReconciledPutIsLoggedAsTheUpdateItIs() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(One), Note("authoritative")); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + var published = new List(); + database.Watch().Subscribe(change => published.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("predicted")); + await database.ApplyPutAsync(message); + + Assert.That(published, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(database.GetMutationLog(shard.ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated }), + "a prediction is never logged and its reconciliation is an ordinary update"); + } + + [Test] + public async Task AnAdmissionRacingDisposalLogsNothingAfterTheDatabaseIsDisposed() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var entered = new System.Threading.ManualResetEventSlim(); + CultNetDatabase? database = null; + // Registered before the database, so it runs first inside the writer's hold, while another thread disposes the database. + using var holdUntilDisposed = cache.AddJournal(_ => + { + entered.Set(); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + try + { + database!.GetLatestMutationLogSequence(ShardId); + } + catch (ObjectDisposedException) + { + return; + } + + System.Threading.Thread.Sleep(1); + } + }); + database = Database(cache, primary: true, store); + var writer = Task.Run(() => database.PutAsync(One, Note("one"))); + Assert.That(entered.Wait(TimeSpan.FromSeconds(5)), Is.True); + + database.Dispose(); + await writer; + + Assert.That(Sequences(store), Is.Empty, "the admission was journaled after disposal began and must not reach the durable log"); + } + + [Test] + public void ABatchAcrossTwoShardsWhoseStoreRefusesReachesTheWriterAsOneExceptionNamingBothShards() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = + [ + new CultNetShardDescriptor("shard-a", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "a:"), + new CultNetShardDescriptor("shard-b", "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)], keyPrefix: "b:") + ], + MutationLogStore = store + }); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("a"), new CultRecordHandle(new CultRecordKey("a:1"))); + batch.Upsert(Note("b"), new CultRecordHandle(new CultRecordKey("b:1"))); + }))!; + + Assert.That(failure.Burned, Is.EqualTo(new[] { new CultNetBurnedSequence("shard-a", 1), new CultNetBurnedSequence("shard-b", 1) })); + } + + // The cache aggregates whatever else fails in the same hold; the documented unwrap finds the log failure inside. + [Test] + public void ALogFailureBesideAnotherFailureInTheSameHoldIsFoundByUnwrappingTheAggregate() + { + var store = new FlakyLogStore { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Database(cache, primary: true, store); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + + var failure = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }))!; + + var logFailure = failure.InnerExceptions.OfType().Single(); + Assert.That(logFailure.Burned.Select(burned => burned.Sequence), Is.EqualTo(new[] { 1L, 2L })); + Assert.That(failure.InnerExceptions.OfType().Single().Message, Is.EqualTo("second journal")); + } + + [Test] + public async Task ABareWriteOfASchemaNoShardOwnsIsNeitherLoggedNorPublishedNorReplicated() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + var database = Database(cache, primary: true, store); + var published = Record(database); + + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "n", Body = "unrelated" }, new CultRecordHandle(new CultRecordKey("unrelated:n"))); + await cache.UpsertAsync(Note("one"), new CultRecordHandle(One)); + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the unrelated write minted no sequence"); + Assert.That(database.GetMutationLog(ShardId).Select(entry => entry.Key.Value), Is.EqualTo(new[] { One.Value })); + Assert.That(published, Is.EqualTo(new[] { One.Value }), "no shard owns the unrelated write, so the database does not publish it"); + } + + // The database is the sharded write door: a schema and key no shard owns cannot be written on any path, and a refused + // write caches and logs nothing. + private static (CultNetDatabase Database, CultCache Cache, FlakyLogStore Store, string SchemaId) OwnedNothingOfNote() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + return (Database(cache, primary: true, store), cache, store, cache.Registry.GetRequired().SchemaId); + } + + private static readonly CultRecordKey Unowned = new("unrelated:note"); + + private static CultNetDocumentPutRawMessage UnownedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "wire" }); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + [Test] + public void ADatabasePutOrDeleteOfASchemaNoShardOwnsIsRefusedNamingTheSchema() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + + var put = Assert.ThrowsAsync(async () => + await database.PutAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "plain" }))!; + Assert.That(put.SchemaId, Is.EqualTo(unowned)); + Assert.That(put.Message, Does.Contain(unowned)); + Assert.That(put, Is.InstanceOf(), "a refused write reads as one"); + Assert.That(cache.Get(Unowned), Is.Null, "nothing was cached"); + Assert.ThrowsAsync(async () => await database.DeleteAsync(Unowned)); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public void ARemoteAuthoritativePutOfASchemaNoShardOwnsIsRefusedAndCachesNothing() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(UnownedPut(database)))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Null); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public async Task ARemoteDeleteOfASchemaNoShardOwnsIsRefusedEvenWhenTheKeyIsCached() + { + var (database, cache, store, unowned) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + + var refusal = Assert.ThrowsAsync(async () => await database.ApplyDeleteAsync(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + }))!; + + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the refused delete removed nothing"); + Assert.That(Sequences(store), Is.Empty); + } + + [Test] + public void APredictionOfASchemaNoShardOwnsIsRefused() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + Shards = [new CultNetShardDescriptor("only-notes", "local", epoch: 1, isPrimary: true, schemaIds: [SchemaId(cache)])], + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + + Assert.ThrowsAsync(async () => + await database.PutPredictedAsync(Unowned, new MeshQuickstartNote { NoteId = "p", Body = "predicted" })); + Assert.That(cache.Get(Unowned), Is.Null); + } + + [Test] + public void ASchemaNoShardOwnsCannotBeWrittenAuthoritativelyAndHasNoShardToResolve() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + + Assert.That(database.CanWriteAuthoritatively(Unowned), Is.False); + Assert.That(database.CanWriteAuthoritatively(One), Is.True, "an owned schema is unaffected"); + var refusal = Assert.Throws(() => database.ResolveShard(unowned, Unowned))!; + Assert.That(refusal.SchemaId, Is.EqualTo(unowned)); + Assert.That(database.ResolveShard(SchemaId(cache), One).ShardId, Is.EqualTo(ShardId)); + } + + // The server answers a refused remote put the way it answers any other: it logs the failure and sends the peer a + // CultNetErrorMessage carrying the refusal. The handler is reached through its private delegate with an + // uninitialized peer, as the R-AM test does; the send then fails on that peer, after the failure was logged. + [Test] + public void TheServerAnswersARemotePutOfASchemaNoShardOwnsWithAnApplicationRejection() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await handler(UnownedPut(database), peer)); + + Assert.That(logger.Warnings, Has.Some.Contains("raw put refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty, "a refusal is not a fault"); + Assert.That(cache.Get(Unowned), Is.Null); + } + + // One cached row of a schema no shard owns is not part of what this database serves: the selection answers with the + // rest, it does not fail as a whole. + [Test] + public async Task ASelectionSkipsACachedRowNoShardOwnsAndAnswersWithTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSelectionResponse(new CultNetSnapshotRequestV1Message + { + MessageId = "select-all", + Selection = new CultNetSelection { Projection = CultNetSelectionProjections.Document } + }); + + Assert.That(response.Documents!.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + Assert.That(response.Matched, Is.EqualTo(1u)); + } + + [CultDocument("tests.journal_indexed", "tests.journal_indexed.v1")] + [MessagePack.MessagePackObject] + public sealed class JournalIndexedNote + { + [MessagePack.Key(0)] + [CultName] + public string Name { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultIndex("tag")] + public string Tag { get; set; } = string.Empty; + } + + // A database serves only what its shards own: reads, the legacy and shard-less snapshot, selections, subscription + // snapshots and the watch agree. The cache behind it still holds the row. + [Test] + public async Task ADatabaseReadsServeOnlyWhatItsShardsOwnWhileTheCacheStillHoldsTheRest() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + var seen = Record(database); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new JournalIndexedNote { Name = "n", Tag = "t" }, new CultRecordHandle(new CultRecordKey("unrelated:indexed"))); + + Assert.That(database.GetAll(), Is.Empty); + Assert.That(database.GetAll().Count(), Is.EqualTo(1), "only the owned row"); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(await database.GetAsync(Unowned), Is.Null); + Assert.That(await database.GetAsync(One), Is.Not.Null); + Assert.That(database.GetByName("u"), Is.Null); + Assert.That(database.GetByName("n"), Is.Null); + Assert.That(database.GetByIndex("tag", "t"), Is.Null); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the cache itself still holds it"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "what the database reads out is what it publishes"); + } + + [Test] + public async Task ADatabaseCollectionOfASchemaNoShardOwnsIsEmpty() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + + var collection = GameCult.Mesh.CultMesh.Collection(database); + + Assert.That(await collection.LatestAsync(), Is.Empty); + } + + [Test] + public async Task TheShardlessSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + + var response = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + + Assert.That(response.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + + [Test] + public async Task ASubscriptionSnapshotOmitsARowNoShardOwns() + { + var (database, cache, _, _) = OwnedNothingOfNote(); + await database.PutAsync(One, Note("owned")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "unowned" }, new CultRecordHandle(Unowned)); + var socket = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Dgram, System.Net.Sockets.ProtocolType.Udp); + socket.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "owned-reads-server", Socket = socket }); + using var subscriptions = new CultNetDatabaseSubscriptionServer(server, database); + using var cancellation = new CancellationTokenSource(); + var serverThread = new Thread(() => + { + while (!cancellation.IsCancellationRequested) + { + _ = server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + serverThread.Start(); + try + { + using var client = CultNetSchemaClients.CreateRudp("owned-reads-client"); + CultNetSnapshotResponseRawMessage? snapshot = null; + client.OnCultNet(message => snapshot = message); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + for (var attempt = 0; attempt < 300 && !client.Connected; attempt++) + await Task.Delay(10); + client.SendCultNet(new CultNetDatabaseSubscribeMessage { MessageId = "subscribe-all", SubscriptionId = "all", IncludeSnapshot = true }); + for (var attempt = 0; attempt < 300 && snapshot == null; attempt++) + await Task.Delay(10); + + Assert.That(snapshot, Is.Not.Null, "the subscription answered a snapshot"); + Assert.That(snapshot!.Documents.Select(document => document.RecordKey), Is.EqualTo(new[] { One.Value })); + } + finally + { + cancellation.Cancel(); + serverThread.Join(); + } + } + + // A replica applies the log the way it applies a snapshot: a row its shard does not own is left out. The entry is + // still recorded, so a replica chained on this one sees no gap. + [Test] + public async Task AReplicaLogApplySkipsARowItsShardDoesNotOwnButRecordsTheEntry() + { + var cache = new CultCache(); + var replica = Database(cache, primary: false, store: null); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "c", Body = "cached" }, new CultRecordHandle(Unowned)); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey("unrelated:put")), new MeshQuickstartNote { NoteId = "p", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + var owned = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("owned")); + owned.ShardId = ShardId; + owned.ShardEpoch = 1; + var now = DateTimeOffset.UtcNow.ToString("O"); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = now, ChangeKind = "added", Put = put }, + new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = cache.Registry.GetRequired().SchemaId, + RecordKey = Unowned.Value, + ShardId = ShardId, + ShardEpoch = 1 + } + }, + new CultNetShardLogEntryMessage { Sequence = 3, CommittedAt = now, ChangeKind = "added", Put = owned } + ] + }); + + Assert.That(cache.Get(new CultRecordKey("unrelated:put")), Is.Null, "the put was left out"); + Assert.That(cache.Get(Unowned), Is.Not.Null, "the delete removed nothing the shard owns"); + Assert.That(TextOf(cache, One), Is.EqualTo("owned"), "owned entries still apply"); + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.That(replica.GetAppliedShardSequence(ShardId), Is.EqualTo(3)); + } + + // The server's put and delete handlers are reached through their private delegates with an uninitialized peer (as + // the R-AM test does): anything the handler tries to send throws on that peer, so a handler that returns quietly + // sent nothing. + private static (CultNetDatabaseServer Server, Func Put, Func Delete, CapturingLogger Logger, Server Host) WireHandlers(CultNetDatabase database, CultCache cache) + { + var host = new Server(cache, ServerSecurityOptions.Development()); + var logger = new CapturingLogger(); + host.Logger = logger; + var databaseServer = new CultNetDatabaseServer(host, database); + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + const System.Reflection.BindingFlags flags = System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance; + var put = (Func)typeof(CultNetDatabaseServer).GetField("_putHandler", flags)!.GetValue(databaseServer)!; + var delete = (Func)typeof(CultNetDatabaseServer).GetField("_deleteHandler", flags)!.GetValue(databaseServer)!; + return (databaseServer, message => put(message, peer), message => delete(message, peer), logger, host); + } + + private static CultNetDocumentPutRawMessage OwnedPut(CultNetDatabase database) + { + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + return message; + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemotePutWhoseLogFailedBesideAnotherFailureIsStillNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = _ => true }); + using var second = cache.AddJournal(_ => throw new InvalidOperationException("second journal")); + var (databaseServer, put, _, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await put(OwnedPut(database)); + + Assert.That(TextOf(cache, One), Is.EqualTo("wire")); + Assert.That(logger.Errors, Is.Empty); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public async Task ACommittedRemoteDeleteWhoseLogFailedIsNotAnsweredAsARefusal() + { + var cache = new CultCache(); + var database = Database(cache, primary: true, new FlakyLogStore { RefuseAppend = sequence => sequence == 2 }); + await database.PutAsync(One, Note("one")); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = SchemaId(cache), + RecordKey = One.Value, + ShardId = ShardId, + ShardEpoch = 1 + }); + + Assert.That(TextOf(cache, One), Is.Null, "the delete committed"); + Assert.That(logger.Warnings, Has.Some.Contains("committed but its shard log failed")); + } + + [Test] + public void ARemoteDeleteOfASchemaNoShardOwnsIsRefusedOnTheWireAtWarningLevel() + { + var (database, cache, _, unowned) = OwnedNothingOfNote(); + var (databaseServer, _, delete, logger, host) = WireHandlers(database, cache); + using var _1 = databaseServer; + using var _2 = host; + + // The handler answers the peer with the refusal; the uninitialized peer cannot send, which is how the test knows it tried. + Assert.CatchAsync(async () => await delete(new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = unowned, + RecordKey = Unowned.Value, + ShardId = database.Shards[0].ShardId, + ShardEpoch = database.Shards[0].Epoch + })); + + Assert.That(logger.Warnings, Has.Some.Contains("raw delete refused").And.Contains(unowned)); + Assert.That(logger.Errors, Is.Empty); + } + + // A change the log holds is published, however Dispose interleaves with the write: Dispose waits for a journal in + // flight (disposing the journal first) and publishes whatever the writer had logged and not yet published. The writer + // is held in an earlier cache observer, so Dispose finishes before the writer reaches the database's own observer. + [Test] + public async Task ADatabaseDisposedWhileAWriteIsInItsLogAppendStillPublishesWhatItLogged() + { + var store = new FlakyLogStore(); + var cache = new CultCache(); + using var inObserver = new ManualResetEventSlim(); + using var releaseObserver = new ManualResetEventSlim(); + using var holdWriter = cache.Watch().Subscribe(_ => + { + inObserver.Set(); + releaseObserver.Wait(TimeSpan.FromSeconds(10)); + }); + var database = Database(cache, primary: true, store); + var seen = Record(database); + using var inAppend = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + store.OnAppend = () => + { + inAppend.Set(); + release.Wait(TimeSpan.FromSeconds(10)); + }; + var writer = Task.Run(async () => await database.PutAsync(One, Note("one"))); + Assert.That(inAppend.Wait(TimeSpan.FromSeconds(10)), Is.True); + + var dispose = Task.Run(database.Dispose); + await Task.Delay(300); + release.Set(); + Assert.That(inObserver.Wait(TimeSpan.FromSeconds(10)), Is.True, "the writer left the gate and is held in the earlier observer"); + await dispose; + releaseObserver.Set(); + await writer; + + Assert.That(Sequences(store), Is.EqualTo(new[] { 1L }), "the write was logged"); + Assert.That(seen, Is.EqualTo(new[] { One.Value }), "and published"); + } + + private sealed class CapturingLogger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public List Warnings { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) => Warnings.Add(message); + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static async Task ApplyAuthoritativeAsync(CultNetDatabase database, CultRecordKey key, string text) + { + var shard = database.Shards[0]; + var message = database.Documents.CreateRawDocumentPutMessage("authoritative", new CultRecordHandle(key), Note(text)); + message.ShardId = shard.ShardId; + message.ShardEpoch = shard.Epoch; + await database.ApplyPutAsync(message); + } + + private static CultNetDatabaseOptions ClientOptions() => new() + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }; + + [Test] + public async Task APredictionOverAPredictionIsStillAPredictionAndTheAuthoritativeWriteReconcilesOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, ClientOptions()); + var kinds = new List(); + database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await database.PutPredictedAsync(One, Note("p1")); + await database.PutPredictedAsync(One, Note("p2")); + await ApplyAuthoritativeAsync(database, One, "authoritative"); + await ApplyAuthoritativeAsync(database, One, "again"); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + [Test] + public async Task AGameSessionPredictionOfTheInstanceTheCacheAlreadyHoldsIsAPredictionThatIsNeverLogged() + { + var rootPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "journal-session", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(rootPath); + using var node = await GameCult.Mesh.CultMesh.CreateNodeAsync(Path.Combine(rootPath, "world.ccmp"), new GameCult.Mesh.CultMeshNodeOptions + { + StartServer = false, + DatabaseOptions = ClientOptions() + }); + using var session = GameCult.Mesh.CultMesh.CreateGameSession(node, new GameCult.Mesh.CultMeshGameSessionOptions + { + ServeSimulationObservations = false, + ServeVerseDiscovery = false, + ServePeerExchange = false + }); + var kinds = new List(); + node.Database.Watch().Subscribe(change => kinds.Add(change.Kind)); + + await session.PredictAsync(One, Note("p1")); + var held = node.Cache.Get(One)!; + held.Text = "p2"; + await session.PredictAsync(One, held); + await ApplyAuthoritativeAsync(node.Database, One, "authoritative"); + + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(node.Database.GetMutationLog(node.Database.Shards[0].ShardId).Select(entry => entry.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + [Test] + public async Task ADatabaseRecordsEveryDeleteEntryItAppliesWhileAnotherThreadRemovesTheSameKeys() + { + const int rounds = 20000; + var replicaCache = new CultCache(); + var replica = Database(replicaCache, primary: false, store: null); + var schemaId = SchemaId(replicaCache); + for (var i = 1; i <= rounds; i++) + await replicaCache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey("race:" + i))); + + var current = 0; + var stop = false; + var remover = Task.Run(() => + { + var next = 1; + while (next <= rounds && !Volatile.Read(ref stop)) + { + if (Volatile.Read(ref current) >= next) + { + replicaCache.Remove(new CultRecordKey("race:" + next)); + next++; + } + } + }); + for (var i = 1; i <= rounds; i++) + { + Volatile.Write(ref current, i); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = ShardId, + ShardEpoch = 1, + Entries = + [ + new CultNetShardLogEntryMessage + { + Sequence = i, + CommittedAt = DateTimeOffset.UtcNow.ToString("O"), + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d" + i, + SchemaId = schemaId, + RecordKey = "race:" + i, + ShardId = ShardId, + ShardEpoch = 1 + } + } + ] + }); + } + + Volatile.Write(ref stop, true); + await remover; + + Assert.That(replica.GetMutationLog(ShardId).Select(entry => entry.Sequence), Is.EqualTo(Enumerable.Range(1, rounds).Select(i => (long)i)), + "a replica's log holds every entry of the primary's, whoever removed the key first"); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs new file mode 100644 index 00000000..62d31e69 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublicationOrderTests.cs @@ -0,0 +1,351 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + public sealed class CultNetPublicationOrderTests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + private static string? TextOf(object? document) => (document as NetworkSchemaNote)?.Text; + + // A1: two writers to one key. A is admitted first, B second; the cache ends at B. A's publication is delayed + // (the window between leaving the cache gate and reaching the handler), B runs to completion meanwhile. + [Test] + public async Task S1_TheLogOrdersOneKeysWritesInAdmissionOrder() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "A") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache); + var seen = new List(); + using var subscriber = database.Watch().Subscribe(change => { lock (seen) seen.Add(change.Document?.Text); }); + var key = new CultRecordKey("soul:order"); + + var a = Task.Run(() => cache.UpsertAsync(Note("A"), new CultRecordHandle(key)).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True); + await database.PutAsync(key, Note("B")); + release.Set(); + await a; + + var shard = database.Shards[0].ShardId; + var last = database.GetMutationLog(shard).Where(entry => entry.Key.Value == key.Value).OrderBy(entry => entry.Sequence).Last(); + + // A replica built from the log. + var replicaCache = new CultCache(); + var replica = new CultNetDatabase(replicaCache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shard, "other", epoch: database.Shards[0].Epoch, isPrimary: false)] + }); + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shard, + ShardEpoch = database.Shards[0].Epoch, + Entries = database.GetMutationLogMessages(shard).ToArray() + }); + + Assert.Multiple(() => + { + Assert.That(cache.Get(key)!.Text, Is.EqualTo("B"), "the cache admitted A then B"); + Assert.That(TextOf(last.Document), Is.EqualTo("B"), "the log's last write for the key must be the cache's last write"); + Assert.That(replicaCache.Get(key)?.Text, Is.EqualTo("B"), "a replica built from the log must match the primary"); + // Delivery is not ordered across threads (order is data: a wire peer drops the stale change by Sequence, Cut 6), + // so a subscriber sees both changes, once each, in either order. + Assert.That(seen, Is.EquivalentTo(new[] { "A", "B" }), "each change is published exactly once"); + }); + } + + // Same property, no instrumentation: two writers hammer one key, then the log's last entry is compared with the cache. + [Test] + public void S1b_UninstrumentedOneKeyRaceLeavesLogAndCacheAgreeing() + { + var divergent = 0; + const int rounds = 300; + for (var round = 0; round < rounds; round++) + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:hammer"); + using var start = new Barrier(2); + Parallel.For(0, 2, writer => + { + start.SignalAndWait(); + for (var i = 0; i < 40; i++) + cache.UpsertAsync(Note($"{writer}:{i}"), new CultRecordHandle(key)).GetAwaiter().GetResult(); + }); + var last = database.GetMutationLog(database.Shards[0].ShardId).OrderBy(entry => entry.Sequence).Last(); + if (!ReferenceEquals(last.Document, cache.Get(key))) + divergent++; + } + + TestContext.Progress.WriteLine($"SOUL S1b divergent rounds: {divergent}/{rounds}"); + Assert.That(divergent, Is.EqualTo(0), $"{divergent}/{rounds} rounds ended with the log's last write differing from the cache"); + } + + // A2: a door's context is keyed by instance. A plain removal of the instance the door just admitted, landing in + // the window before the door's own change is handled, consumes the door's context. + [Test] + public async Task S2_ARemovalNeverWearsAReconcilingPutsContext() + { + var cache = new CultCache(); + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + using var widen = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + entered.Set(); + release.Wait(Wait); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:reconcile"); + await database.PutPredictedAsync(key, Note("predicted")); + var published = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Text)>(); + using var subscriber = database.Watch().Subscribe(change => published.Enqueue((change.Kind, change.Document?.Text))); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + var a = Task.Run(() => database.ApplyPutAsync(message)); + Assert.That(entered.Wait(Wait), Is.True); + cache.Remove(key); + release.Set(); + await a; + + var log = database.GetMutationLog(database.Shards[0].ShardId, logged).ToArray(); + Assert.Multiple(() => + { + Assert.That(published.Count(change => change.Kind == CultNetDatabaseChangeKind.Removed), Is.EqualTo(1), + "the removal is published as a removal: " + string.Join(", ", published)); + Assert.That(published.Where(change => change.Kind == CultNetDatabaseChangeKind.Reconciled).Select(change => change.Text), + Is.EqualTo(new[] { "authoritative" }), "the reconcile carries the authoritative document"); + Assert.That(log.Where(entry => entry.Kind != CultNetDatabaseChangeKind.Removed).All(entry => entry.Document != null), Is.True, + "no put entry without a document: " + string.Join(", ", log.Select(entry => entry.Kind + ":" + TextOf(entry.Document)))); + }); + } + + // A4: chained replication. B is a replica of P, C a replica of B. A replica-applied delete of a key B does not + // hold (B removed it locally, a supported write) must still keep B's log contiguous for C. + [Test] + public async Task S3_AChainedReplicaSeesNoGapWhenItsSourceAppliesADeleteOfAnAbsentKey() + { + const string shardId = "chain"; + var schemaId = new CultCache().Registry.GetRequired().SchemaId; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [schemaId])] + }); + var p = Make(new CultCache(), true); + var bCache = new CultCache(); + var b = Make(bCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId)).ToArray() + }); + } + + var k = new CultRecordKey("soul:chain:k"); + await p.PutAsync(k, Note("x")); + await Pull(p, b); + await Pull(b, c); + bCache.Remove(k); + await p.DeleteAsync(k); + await p.PutAsync(new CultRecordKey("soul:chain:k2"), Note("y")); + await Pull(p, b); + + Assert.Multiple(() => + { + Assert.That(b.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L }), + "B's log holds every primary entry it applied"); + Assert.DoesNotThrowAsync(() => Pull(b, c), "C pulls B's log without a gap"); + }); + } + + // A5: a subscriber that answers a Reconciled change by predicting the same key again keeps its prediction. + [Test] + public async Task S4_APredictionMadeInReactionToAReconcileSurvives() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:repredict"); + var kinds = new ConcurrentQueue(); + var repredicted = false; + using var subscriber = database.Watch().Subscribe(change => + { + kinds.Enqueue(change.Kind); + if (change.Kind == CultNetDatabaseChangeKind.Reconciled && !repredicted) + { + repredicted = true; + database.PutPredictedAsync(key, Note("predicted again")).GetAwaiter().GetResult(); + } + }); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled + })); + } + // Target of AWriteAfterADatabaseDeleteIsStillPublishedAndLogged: a door context must not outlive its door. A + // refused prediction leaves nothing behind that a later write of the same instance would wear. + [Test] + public async Task S6_ARefusedDoorLeavesNoContextForALaterWriteOfTheSameInstance() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var baseKey = new CultRecordKey("soul-refuse-base"); + var variantKey = new CultRecordKey("soul-refuse-variant"); + await cache.UpsertAsync(new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "base", Power = 1 }, + new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(CultNetDocumentRegistryVariantTests.VariantWireFixture.Name), "v") + }); + var instance = new CultNetDocumentRegistryVariantTests.VariantWireFixture { Name = "reused", Power = 3 }; + Assert.That(async () => await database.PutPredictedAsync(variantKey, instance), Throws.InvalidOperationException); + var kinds = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => kinds.Enqueue(change.Kind)); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await cache.UpsertAsync(instance, new CultRecordHandle(new CultRecordKey("soul-refuse-other"))); + + Assert.Multiple(() => + { + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Added })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged), Has.Count.EqualTo(1)); + }); + } + + // Target of ADisposedDatabaseNoLongerObservesTheCache: after Dispose the database must stop logging, which a + // durable log store can observe even though the disposed Subject hides publication. + [Test] + public async Task S7_ADisposedDatabaseAppendsNothingToItsDurableLog() + { + var root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "soul-log-" + Guid.NewGuid().ToString("N")); + var store = new CultNetFileShardMutationLogStore(root); + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { MutationLogStore = store }); + var shard = database.Shards[0].ShardId; + await cache.UpsertAsync(Note("before"), new CultRecordHandle(new CultRecordKey("soul:dispose:before"))); + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + database.Dispose(); + + try { await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("soul:dispose:after"))); } + catch { /* a throwing handler would also be a failure below */ } + + Assert.That(store.Read(shard), Has.Count.EqualTo(1)); + System.IO.Directory.Delete(root, true); + } + + // The log entry exists before its change is published: a subscriber that reads the log from its handler finds it. + [Test] + public async Task TheLogEntryIsWrittenBeforeTheChangeIsPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("soul:log-first"); + var loggedWhenPublished = new ConcurrentQueue(); + using var subscriber = database.Watch().Subscribe(change => + loggedWhenPublished.Enqueue(database.GetMutationLog(database.Shards[0].ShardId).Any(entry => entry.Key.Value == key.Value))); + + await database.PutAsync(key, Note("one")); + cache.Remove(key); + + Assert.That(loggedWhenPublished, Is.EqualTo(new[] { true, true })); + } + + // An observer that mutates the admitted instance in place and upserts it again writes a second change of the same + // instance. It is published as its own update and never wears the context of the door's change, even when it + // reaches the database first. + [Test] + public async Task AnInPlaceReUpsertOfTheDoorsInstanceDoesNotWearTheDoorsContext() + { + var cache = new CultCache(); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "authoritative") + { + change.Document.Text = "normalized"; + cache.UpsertAsync(change.Document, new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + } + }); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul:in-place"); + var seen = new ConcurrentQueue<(CultNetDatabaseChangeKind Kind, string? Previous)>(); + using var subscriber = database.Watch().Subscribe(change => seen.Enqueue((change.Kind, change.PreviousDocument?.Text))); + var message = database.Documents.CreateRawDocumentPutMessage("m", new CultRecordHandle(key), Note("authoritative")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(message); + + // The observer's nested write is delivered depth-first, inside the delivery of the change that caused it, so + // the database may publish the two in either order; what is pinned is that each carries its own context. + Assert.That(seen, Is.EquivalentTo(new (CultNetDatabaseChangeKind, string?)[] + { + (CultNetDatabaseChangeKind.Predicted, null), + (CultNetDatabaseChangeKind.Reconciled, "predicted"), + (CultNetDatabaseChangeKind.Updated, "normalized") + })); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs new file mode 100644 index 00000000..fbd1df2a --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetPublishesCommitsTests.cs @@ -0,0 +1,429 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; +using static GameCult.Networking.Tests.CultNetDocumentRegistryVariantTests; + +namespace GameCult.Networking.Tests +{ + // The cache is the one owner of "a change was committed": whatever door wrote it, a subscriber over the wire and the + // mutation log see it once. A database write door publishes its own change (it carries the shard-log entry); every + // other change in the hold publishes from the cache. + public sealed class CultNetPublishesCommitsTests + { + private static CultNetDatabaseChangeRawMessage[] Of(Wire wire, string key) => + wire.Changes.Where(change => KeyOf(change) == key).ToArray(); + + private static string? KeyOf(CultNetDatabaseChangeRawMessage change) => change.Document?.RecordKey ?? change.RecordKey; + + private sealed class Wire : IDisposable + { + private readonly CancellationTokenSource _cancellation = new(); + private readonly Thread _thread; + private readonly RudpCultNetSchemaServer _server; + private readonly CultNetDatabaseSubscriptionServer _subscriptions; + private readonly ICultNetSchemaClient _client; + public ConcurrentQueue Changes { get; } = new(); + public int Snapshots; + public ConcurrentQueue Errors { get; } = new(); + + private readonly CultNetDatabase _database; + + public Wire(CultNetDatabase database) + { + _database = database; + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + _server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { RuntimeId = "publish-server", Socket = socket }); + _subscriptions = new CultNetDatabaseSubscriptionServer(_server, database); + _thread = new Thread(() => + { + while (!_cancellation.IsCancellationRequested) + { + _ = _server.PollOnceAsync().GetAwaiter().GetResult(); + Thread.Sleep(1); + } + }) { IsBackground = true }; + _thread.Start(); + _client = CultNetSchemaClients.CreateRudp("publish-client"); + _client.OnCultNet(Changes.Enqueue); + _client.OnCultNet(Errors.Enqueue); + _client.OnCultNet(_ => Interlocked.Increment(ref Snapshots)); + _client.Connect("127.0.0.1", _server.LocalEndPoint.Port); + } + + public async Task SubscribeAsync(bool includeSnapshot, params Type[] documentTypes) + { + await Until(() => _client.Connected); + _client.SendCultNet(new CultNetDatabaseSubscribeMessage + { + MessageId = "subscribe", + SubscriptionId = "all", + SchemaIds = documentTypes.Select(type => _database.Cache.Registry.GetRequired(type).SchemaId).ToArray(), + IncludeSnapshot = includeSnapshot + }); + await Until(() => Snapshots > 0 || !Errors.IsEmpty); + Assert.That(Errors.Select(error => error.Error), Is.Empty, "the subscription was refused"); + } + + public void Dispose() + { + _cancellation.Cancel(); + _thread.Join(); + _client.Dispose(); + _subscriptions.Dispose(); + _server.Dispose(); + } + } + + private static async Task Until(Func condition) + { + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + Assert.Fail("Condition was not satisfied before timeout."); + await Task.Delay(5); + } + } + + // One reliable ordered channel: a sentinel write that has arrived proves every earlier write has had its chance + // to arrive twice. + private static async Task SettleAsync(CultNetDatabase database, Wire wire) + { + var sentinel = "tests:publish:sentinel:" + Guid.NewGuid().ToString("N"); + await database.PutAsync(new CultRecordKey(sentinel), new NetworkSchemaNote { Schema = "tests.networking_note.v1" }); + await Until(() => wire.Changes.Any(change => KeyOf(change) == sentinel)); + } + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + + [Test] + public async Task ACacheCommitAndUpsertReachASubscriberOnce() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:commit"); + + cache.Commit(batch => batch.Upsert(Note("committed"), new CultRecordHandle(key))); + await Until(() => Of(wire, key.Value).Length >= 1); + cache.Commit(batch => batch.Remove(key)); + await Until(() => Of(wire, key.Value).Length >= 2); + await cache.UpsertAsync(Note("upserted"), new CultRecordHandle(key)); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed", "added" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(3)); + } + + [Test] + public async Task ADatabasePutAndDeleteStillPublishExactlyOnce() + { + var database = new CultNetDatabase(new CultCache()); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:put"); + + await database.PutAsync(key, Note("put")); + await database.DeleteAsync(key); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + [Test] + public async Task AReplicatedWriteIsPublishedAndLoggedOnceNotEchoed() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:replicated"); + var message = database.Documents.CreateRawDocumentPutMessage( + "put-replicated", new CultRecordHandle(key), Note("remote")); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + + await database.ApplyPutAsync(message); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value), Has.Length.EqualTo(1)); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(1)); + } + + [Test] + public async Task ALoadRemovalCarriesItsKey() + { + var path = Path.Combine(Path.GetTempPath(), $"cultnet-load-{Guid.NewGuid():N}.cc"); + try + { + var key = new CultRecordKey("tests:publish:loaded"); + var writer = new CultCache(); + writer.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await writer.UpsertAsync(Note("there"), new CultRecordHandle(key)); + writer.FlushAllBackingStores(); + + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + var database = new CultNetDatabase(cache); + Assert.That(cache.Get(key), Is.Not.Null); + using var wire = new Wire(database); + await wire.SubscribeAsync(true, typeof(NetworkSchemaNote)); + + writer.Remove(key); + writer.FlushAllBackingStores(); + await cache.PullAllBackingStoresAsync(); + await SettleAsync(database, wire); + + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "removed" })); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(path + ".lock")) File.Delete(path + ".lock"); + } + } + + [Test] + public async Task ABaseEditPublishesItsVariantDependentAndTheSubscriberIsRefusedItByKey() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big"), + cache.Override(nameof(VariantWireFixture.Power), 9) + }); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(VariantWireFixture)); + var errorsBefore = wire.Errors.Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + await Until(() => wire.Errors.Count > errorsBefore); + await Until(() => Of(wire, baseKey.Value).Length >= 1); + + Assert.That(wire.Errors.Skip(errorsBefore).Single().Error, Does.Contain(variantKey.Value), + "the re-resolved dependent is published, then refused (Q6) by key"); + Assert.That(Of(wire, baseKey.Value), Has.Length.EqualTo(1)); + Assert.That(Of(wire, variantKey.Value), Is.Empty, "a variant is never carried as a record"); + } + + // An observer that rewrites the very key a door just wrote is a second write: it publishes and logs on its own. + [Test] + public async Task AnObserverRewriteOfTheDoorsOwnKeyIsPublishedAndLoggedToo() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + using var wire = new Wire(database); + await wire.SubscribeAsync(false, typeof(NetworkSchemaNote)); + var key = new CultRecordKey("tests:publish:normalized"); + using var normalizer = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "raw") + cache.UpsertAsync(Note("normalized"), new CultRecordHandle(change.Key)).GetAwaiter().GetResult(); + }); + + await database.PutAsync(key, Note("raw")); + await Until(() => Of(wire, key.Value).Length >= 2); + await SettleAsync(database, wire); + + Assert.That(cache.Get(key)!.Text, Is.EqualTo("normalized")); + Assert.That(Of(wire, key.Value).Select(change => change.ChangeKind), Is.EqualTo(new[] { "added", "updated" })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Count(entry => entry.Key.Value == key.Value), Is.EqualTo(2)); + } + + private static (CultCache Cache, CultNetDocumentRegistry Documents) VariantCache() + { + var registry = CultDocumentRegistry.Shared; + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + var documents = new CultNetDocumentRegistry(registry) + .Register(CultNetDocumentBinding.ForDocument(registry)); + return (cache, documents); + } + + private static System.Collections.Generic.List Record(CultNetDatabase database, Func describe) + { + var seen = new System.Collections.Generic.List(); + database.WatchAllChanges().Subscribe(change => seen.Add(describe(change))); + return seen; + } + + private static string KeyOfChange(object change) => ((dynamic)change).Key.Value; + + [Test] + public async Task ABaseWriteAndItsReResolvedVariantAreLoggedInCacheOrder() + { + var (cache, documents) = VariantCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { DocumentRegistry = documents }); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await cache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await cache.UpsertVariantAsync(variantKey, baseKey, new[] + { + cache.Override(nameof(VariantWireFixture.Name), "big") + }); + var published = Record(database, KeyOfChange); + var logged = database.GetMutationLog(database.Shards[0].ShardId).Count; + + await database.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + Assert.That(published, Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId, logged).Select(entry => entry.Key.Value), + Is.EqualTo(new[] { baseKey.Value, variantKey.Value })); + } + + private static CultNetDatabase Replica(CultCache cache, CultNetDocumentRegistry? documents, string schemaId) => + new(cache, new CultNetDatabaseOptions + { + DocumentRegistry = documents, + Shards = + [ + new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: false, schemaIds: [schemaId]) + ] + }); + + [Test] + public async Task AReplicaLogHoldsOnlyThePrimarysSequencesWhateverItResolvesLocally() + { + var (sourceCache, sourceDocuments) = VariantCache(); + var (replicaCache, replicaDocuments) = VariantCache(); + var schemaId = sourceCache.Registry.GetRequired().SchemaId; + var source = new CultNetDatabase(sourceCache, new CultNetDatabaseOptions + { + DocumentRegistry = sourceDocuments, + Shards = [new CultNetShardDescriptor("replica-shard", "runtime-a", epoch: 3, isPrimary: true, schemaIds: [schemaId])] + }); + var replica = Replica(replicaCache, replicaDocuments, schemaId); + var baseKey = new CultRecordKey("publish-variant-base"); + var variantKey = new CultRecordKey("publish-variant-big"); + await replicaCache.UpsertAsync(new VariantWireFixture { Name = "base", Power = 1 }, new CultRecordHandle(baseKey)); + await replicaCache.UpsertVariantAsync(variantKey, baseKey, new[] + { + replicaCache.Override(nameof(VariantWireFixture.Name), "big") + }); + await source.PutAsync(baseKey, new VariantWireFixture { Name = "base edited", Power = 2 }); + + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Entries = source.GetMutationLogMessages("replica-shard").ToArray() + }); + + Assert.That(replicaCache.Get(variantKey)!.Power, Is.EqualTo(2), "the dependent re-resolved"); + Assert.That(replica.GetMutationLog("replica-shard").Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L }), + "the replica log holds the primary's entry and mints none of its own"); + } + + [Test] + public async Task ALocalCacheWriteOnAReplicaIsPublishedOnceAndNeverLogged() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var published = Record(replica, KeyOfChange); + + await cache.UpsertAsync(Note("local"), new CultRecordHandle(new CultRecordKey("tests:publish:local"))); + + Assert.That(published, Is.EqualTo(new[] { "tests:publish:local" })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public void ConcurrentWritersLoseNoLogEntryAndReuseNoSequence() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + const int writers = 4, perWriter = 5000; + + Parallel.For(0, writers, writer => + { + for (var i = 0; i < perWriter; i++) + cache.UpsertAsync(Note("n"), new CultRecordHandle(new CultRecordKey($"tests:publish:c:{writer}:{i}"))) + .GetAwaiter().GetResult(); + }); + + var sequences = database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Sequence).ToArray(); + Assert.That(sequences, Has.Length.EqualTo(writers * perWriter)); + Assert.That(sequences.Distinct().Count(), Is.EqualTo(writers * perWriter)); + } + + [Test] + public async Task AWriteAfterADatabaseDeleteIsStillPublishedAndLogged() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var key = new CultRecordKey("tests:publish:after-delete"); + await database.PutAsync(key, Note("one")); + await database.DeleteAsync(key); + var published = Record(database, KeyOfChange); + + await cache.UpsertAsync(Note("two"), new CultRecordHandle(key)); + + Assert.That(published, Is.EqualTo(new[] { key.Value })); + Assert.That(database.GetMutationLog(database.Shards[0].ShardId).Select(entry => entry.Kind), + Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Added, CultNetDatabaseChangeKind.Removed, CultNetDatabaseChangeKind.Added + })); + } + + [Test] + public async Task AShardSnapshotResyncPublishesItsRemovalsAndPutsOnceAndLogsNothing() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var replica = Replica(cache, null, schemaId); + var stale = new CultRecordKey("tests:publish:stale"); + var fresh = new CultRecordKey("tests:publish:fresh"); + await cache.UpsertAsync(Note("stale"), new CultRecordHandle(stale)); + var published = Record(replica, change => ((dynamic)change).Kind.ToString() + ":" + ((dynamic)change).Key.Value); + var put = replica.Documents.CreateRawDocumentPutMessage( + "snapshot-fresh", new CultRecordHandle(fresh), Note("fresh")); + + await replica.ApplyShardSnapshotResponseAsync(replica.Shards[0], new CultNetSnapshotResponseRawMessage + { + ShardId = "replica-shard", + ShardEpoch = 3, + Documents = [put.Document!], + ShardLogSequence = 5 + }); + + Assert.That(published, Is.EqualTo(new[] { "Removed:" + stale.Value, "Added:" + fresh.Value })); + Assert.That(replica.GetMutationLog("replica-shard"), Is.Empty); + } + + [Test] + public async Task ADisposedDatabaseNoLongerObservesTheCache() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var published = Record(database, KeyOfChange); + database.Dispose(); + + await cache.UpsertAsync(Note("after"), new CultRecordHandle(new CultRecordKey("tests:publish:disposed"))); + + Assert.That(published, Is.Empty); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs new file mode 100644 index 00000000..87249dd4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs @@ -0,0 +1,840 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using GameCult.Mesh; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A Connect's sequence says whether it repeats the one a session accepted, and the handshake + /// alone seeds the watermark that orders delivery. + /// + public sealed class CultNetRudpConnectGenerationTests + { + private const uint ConnectionId = 0x10203090; + private static readonly CultNetRudpSendOptions Ordered = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + + private static CultNetRudpSession Session(uint initialSequence) => + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = initialSequence }); + + private static CultNetRudpPacket Send(CultNetRudpSession session, string text, bool ordered = true, string channel = "schema") => + session.Send(channel, Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = ordered }); + + private static string[] Names(CultNetRudpReceiveResult result) => + result.Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(); + + private static (CultNetRudpPacket Connect, CultNetRudpPacket Accept) Handshake(CultNetRudpSession client, CultNetRudpSession server) + { + var connect = client.CreateConnect(0); + var accept = server.AcceptConnect(connect, 0); + client.Receive(accept, 0); + Assert.That(client.Connected && server.Connected, Is.True); + return (connect, accept); + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + [Test] + public void AReconnectOnTheSameSessionIsNotStrandedByAWriteTheEndedSessionOwed() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + Send(server, "lost"); + server.CreateDisconnect(Encoding.UTF8.GetBytes("gone")); + + client.Receive(server.AcceptConnect(client.CreateConnect(1), 1), 1); + Assert.That(Names(client.Receive(Send(server, "after"), 2)), Is.EqualTo(new[] { "after" })); + } + + [Test] + public void AServerThatAcceptsANewClientForgetsTheOldClientsSequences() + { + var oldClient = Session(50); + var server = Session(500); + Handshake(oldClient, server); + foreach (var name in new[] { "c1", "c2", "c3" }) + Assert.That(Names(server.Receive(Send(oldClient, name), 1)), Is.EqualTo(new[] { name })); + server.CreateDisconnect(Encoding.UTF8.GetBytes("restart")); + + var newClient = Session(2); + newClient.Receive(server.AcceptConnect(newClient.CreateConnect(2), 2), 2); + Assert.That(Names(server.Receive(Send(newClient, "fresh"), 3)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void ARetransmittedConnectAfterDataHasFlowedDoesNotResetTheSession() + { + var client = Session(1); + var server = Session(500); + var (connect, _) = Handshake(client, server); + var frame = Send(client, "once"); + Assert.That(Names(server.Receive(frame, 1)), Is.EqualTo(new[] { "once" })); + + Assert.That(server.ConnectRepeats(connect), Is.True); + Assert.That(server.AcceptConnect(connect, 2).PacketType, Is.EqualTo(CultNetRudpPacketType.Ack), "the Accept was already acknowledged"); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(0), "a repeat queued something"); + Assert.That(server.Receive(frame, 3).Delivered, Is.Empty, "a retransmitted Connect made the server forget what it had delivered"); + } + + [Test] + public void AConnectWithAnotherSequenceIsNotARepeatWhateverItCarries() + { + var server = Session(500); + var connect = Session(1).CreateConnect(0, Encoding.UTF8.GetBytes("same payload")); + server.AcceptConnect(connect, 0); + Assert.That(server.ConnectRepeats(connect), Is.True); + + Assert.That(server.ConnectRepeats(Session(9).CreateConnect(0, Encoding.UTF8.GetBytes("same payload"))), Is.False); + var notAConnect = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(connect)); + notAConnect.PacketType = CultNetRudpPacketType.Ping; + Assert.That(server.ConnectRepeats(notAConnect), Is.False); + server.CreateDisconnect(); + Assert.That(server.ConnectRepeats(connect), Is.False, "an ended session repeats nothing"); + } + + [Test] + public void AnAcceptThatDoesNotNameThePendingConnectIsIgnored() + { + var client = Session(10); + var server = Session(500); + var connect = client.CreateConnect(0); + var accept = server.AcceptConnect(connect, 0); + + var stale = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + stale.Ack = connect.Sequence + 50; + stale.AckMask = 0; + client.Receive(stale, 1); + Assert.That(client.Connected, Is.False, "a stale Accept connected the session"); + Assert.That(client.PendingReliableSequences, Is.EqualTo(new[] { connect.Sequence })); + + var named = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + named.Ack = connect.Sequence + 3; + named.AckMask = 1u << 2; + client.Receive(named, 2); + Assert.That(client.Connected, Is.True, "the mask names the Connect"); + } + + [Test] + public void AnAcceptAfterALocalDisconnectDoesNotReconnectTheSession() + { + var client = Session(1); + var server = Session(500); + var (_, accept) = Handshake(client, server); + client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")); + Assert.That(client.Connected, Is.False); + + client.Receive(accept, 5); + Assert.That(client.Connected, Is.False, "a late Accept revived an ended session"); + } + + [Test] + public void AReconnectForgetsWhatTheOldServerSent() + { + var client = Session(1); + var oldServer = Session(500); + Handshake(client, oldServer); + foreach (var name in new[] { "d1", "d2", "d3" }) + Assert.That(Names(client.Receive(Send(oldServer, name), 1)), Is.EqualTo(new[] { name })); + + var newServer = Session(501); + client.Receive(newServer.AcceptConnect(client.CreateConnect(2), 2), 2); + Assert.That(Names(client.Receive(Send(newServer, "fresh"), 3)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void AnAcceptForAnAbandonedConnectDoesNotConnectTheSession() + { + var client = Session(1); + var server = Session(500); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + client.CreateDisconnect(Encoding.UTF8.GetBytes("never mind")); + + client.Receive(accept, 1); + Assert.That(client.Connected, Is.False, "an Accept revived an abandoned Connect"); + } + + [Test] + public void AFullQueueOwedToAVanishedPeerDoesNotRefuseANewGeneration() + { + static CultNetRudpSession Bounded(uint initialSequence) => new CultNetRudpSession(new CultNetRudpSessionOptions + { + ConnectionId = ConnectionId, + InitialSequence = initialSequence, + MaxPendingReliablePackets = 1 + }); + + var server = Bounded(500); + server.AcceptConnect(Session(1).CreateConnect(0), 0); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(1), "the Accept is owed and never acknowledged"); + var next = Session(9).CreateConnect(1); + server.AcceptConnect(next, 1); + Assert.That(server.ConnectRepeats(next), Is.True); + + var client = Bounded(1); + client.CreateConnect(0); + client.CreateConnect(1); + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(1), "only the new Connect is owed"); + } + + [Test] + public void ADuplicateAcceptDoesNotReseedTheWatermark() + { + var client = Session(1); + var server = Session(500); + var (_, accept) = Handshake(client, server); + var s1 = Send(server, "s1"); + var s2 = Send(server, "s2"); + var forged = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + forged.Sequence = s2.Sequence; + client.Receive(forged, 1); + Assert.That(client.Receive(s2, 2).Delivered, Is.Empty, "s1 is still missing"); + Assert.That(Names(client.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" })); + } + + [Test] + public void ReliableDataBeforeTheAcceptIsNeitherDeliveredNorAcknowledged() + { + var client = Session(1); + var server = Session(500); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + var a = Send(server, "A"); + var b = Send(server, "B"); + + foreach (var early in new[] { b, a }) + { + Assert.That(client.Receive(early, 1).Delivered, Is.Empty); + var ack = client.CreateAckForReceived(early.Sequence); + Assert.That((ack.Ack, ack.AckMask), Is.EqualTo((0u, 0u)), "an unhandled packet was acknowledged"); + } + client.Receive(accept, 2); + Assert.That(client.Connected, Is.True); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(3), "nothing was acknowledged"); + + Assert.That(Names(client.Receive(a, 3)), Is.EqualTo(new[] { "A" })); + Assert.That(Names(client.Receive(b, 3)), Is.EqualTo(new[] { "B" })); + } + + [Test] + public void OrderedFramesAfterTheAcceptAreDeliveredInOrderWhateverArrivesFirst() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + var a = Send(server, "A"); + var b = Send(server, "B"); + Assert.That(client.Receive(b, 1).Delivered, Is.Empty); + Assert.That(Names(client.Receive(a, 2)), Is.EqualTo(new[] { "A", "B" })); + } + + [Test] + public void AnUnorderedReliableFrameIsDeliveredOnReceiptNeverHeldBehindAGap() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + Send(server, "lost"); + var unordered = Send(server, "now", ordered: false, channel: "media"); + var ordered = Send(server, "later"); + Assert.That(Names(client.Receive(unordered, 1)), Is.EqualTo(new[] { "now" })); + Assert.That(client.Receive(ordered, 2).Delivered, Is.Empty, "the ordered frame waits for the gap"); + } + + [Test] + public void APeerDisconnectDoesNotForgetWhatThePeerSent() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + var s1 = Send(client, "s1"); + var s2 = Send(client, "s2"); + Assert.That(server.Receive(s2, 1).Delivered, Is.Empty, "s1 is missing, so s2 is held"); + + server.Receive(client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")), 2); + Assert.That(Names(server.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" }), "the Disconnect forgot the held frame"); + Assert.That(server.Receive(s2, 4).Delivered, Is.Empty, "the Disconnect forgot what was received"); + } + + /// + /// The options say nothing about the initial sequence; each session draws its own, so one options + /// object serves many sessions without giving them one sequence. + /// + [Test] + public void SessionsFromOneOptionsObjectEachDrawTheirOwnInitialSequence() + { + Assert.That(new CultNetRudpSessionOptions().InitialSequence, Is.Null); + Assert.That(new CultNetRudpSocketTransportOptions().InitialSequence, Is.Null); + Assert.That(new CultNetRudpSocketTransportServerOptions().InitialSequence, Is.Null); + Assert.That(new RudpCultNetSchemaServerOptions().InitialSequence, Is.Null); + Assert.That(new CultMeshRudpSocketOptions().InitialSequence, Is.Null); + + var options = new CultNetRudpSessionOptions { ConnectionId = ConnectionId }; + var firsts = Enumerable.Range(0, 64) + .Select(_ => new CultNetRudpSession(options).CreateConnect(0).Sequence) + .ToArray(); + Assert.That(firsts.Distinct().Count(), Is.GreaterThan(32), "a default session does not draw its own sequence"); + Assert.That(firsts.All(value => value >= 1 && value < (1u << 31)), Is.True, "a draw left [1, 2^31)"); + } + + /// + /// A transport built from the same options each time (a reconnect loop's factory) opens each + /// connection with its own Connect sequence. + /// + [Test] + public void AReusedOptionsObjectYieldsADifferentConnectSequenceForEachTransport() + { + using var listener = Bind(); + using var clientSocket = Bind(); + var options = new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = clientSocket, + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = listener.LocalEndPoint, + ConnectionId = ConnectionId + }; + var sequences = new HashSet(); + for (var index = 0; index < 8; index++) + { + var client = new CultNetRudpSocketTransportConnection(options); + client.Connect("hello"); + sequences.Add(Drain(listener).Last(p => p.PacketType == CultNetRudpPacketType.Connect).Sequence); + } + Assert.That(sequences, Has.Count.EqualTo(8), "two connections opened with one Connect sequence"); + } + + /// + /// A listener builds a session for every client it admits; each draws its own sequence, so no two + /// clients are accepted with the same one. + /// + [Test] + public void TheListenerAcceptsEachClientWithItsOwnSequence() + { + using var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var accepts = new HashSet(); + for (var index = 0; index < 8; index++) + { + using var peerSocket = Bind(); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(Session(1).CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + accepts.Add(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept).Sequence); + } + Assert.That(accepts, Has.Count.EqualTo(8), "two clients were accepted with one sequence"); + } + + // Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed Connect from an + // earlier attempt is stale, a Connect from a new endpoint is a new client. + + private static bool NamesSequence(CultNetRudpPacket ack, uint sequence) => + ack.Ack == sequence + || Enumerable.Range(0, 32).Any(bit => (ack.AckMask & (1u << bit)) != 0 && ack.Ack > bit && ack.Ack - (uint)bit - 1 == sequence); + + /// + /// A frame the client's old generation already delivered, retransmitted after the client + /// reconnected, is acknowledged and not delivered into the new generation. + /// + [Test] + public void AFrameFromTheClientsEarlierGenerationIsAcknowledgedNotDeliveredAgain() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var old = Send(client, "old"); + Assert.That(Names(server.Receive(old, 1)), Is.EqualTo(new[] { "old" })); + + var connect = client.CreateConnect(2); + client.Receive(server.AcceptConnect(connect, 2), 2); + Assert.That(old.Sequence, Is.LessThan(connect.Sequence)); + Assert.That(server.Receive(old, 3).Delivered, Is.Empty, "an earlier generation's frame was delivered again"); + Assert.That(NamesSequence(server.CreateAckForReceived(old.Sequence), old.Sequence), Is.True, "the duplicate was not acknowledged"); + Assert.That(Names(server.Receive(Send(client, "fresh"), 4)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void AFrameFromTheServersEarlierGenerationIsAcknowledgedNotDeliveredAgain() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var old = Send(server, "old"); + Assert.That(Names(client.Receive(old, 1)), Is.EqualTo(new[] { "old" })); + + client.Receive(server.AcceptConnect(client.CreateConnect(2), 2), 2); + Assert.That(client.Receive(old, 3).Delivered, Is.Empty, "an earlier generation's frame was delivered again"); + Assert.That(NamesSequence(client.CreateAckForReceived(old.Sequence), old.Sequence), Is.True); + Assert.That(Names(client.Receive(Send(server, "fresh"), 4)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void ADuplicateBelowTheReceiveWindowIsAcknowledgedByName() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var first = Send(client, "first"); + Assert.That(Names(server.Receive(first, 1)), Is.EqualTo(new[] { "first" })); + for (var index = 0; index < 4_200; index++) + { + server.Receive(Send(client, "x"), 1); + client.Receive(server.CreateAck(), 1); + } + Assert.That(server.Receive(first, 2).Delivered, Is.Empty); + Assert.That(server.CreateAckForReceived(first.Sequence).Ack, Is.EqualTo(first.Sequence), "a duplicate below the window was not acknowledged by name"); + } + + /// + /// A Connect delayed from an earlier attempt of the client that is now connected does not + /// restart the session: the server answers with an Ack and the client's ordered frames keep + /// flowing. + /// + [Test] + public void ADelayedConnectFromAnEarlierAttemptDoesNotStrandTheClient() + { + var client = Session(10); + var server = Session(500); + var earlier = client.CreateConnect(0); + var retried = client.CreateConnect(300); + var accept = server.AcceptConnect(retried, 301); + client.Receive(accept, 302); + server.Receive(client.CreateAckForReceived(accept.Sequence), 302); + Assert.That(client.Connected, Is.True); + var a = Send(client, "a"); + Assert.That(Names(server.Receive(a, 303)), Is.EqualTo(new[] { "a" })); + client.Receive(server.CreateAckForReceived(a.Sequence), 303); + + Assert.That(server.ConnectRepeats(earlier), Is.True, "the delayed Connect starts nothing"); + var reply = server.AcceptConnect(earlier, 304); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + client.Receive(reply, 305); + Assert.That(client.Connected, Is.True); + + var b = Send(client, "b"); + Assert.That(Names(server.Receive(b, 306)), Is.EqualTo(new[] { "b" }), "the delayed Connect reset the server"); + client.Receive(server.CreateAckForReceived(b.Sequence), 307); + Assert.That(client.PendingReliableSequences, Does.Not.Contain(b.Sequence)); + } + + /// + /// A stale Connect is the peer's echo from the past, not evidence it is alive; a retransmit of + /// the accepted Connect is. + /// + [Test] + public void ARepeatedConnectRefreshesLivenessAndAStaleOneDoesNot() + { + var client = Session(10); + var server = Session(500); + var earlier = client.CreateConnect(0); + var current = client.CreateConnect(1); + server.AcceptConnect(current, 0); + + server.AcceptConnect(current, 900); + Assert.That(server.CheckTimeout(1_000, 500), Is.False, "a repeated Connect did not refresh liveness"); + + server.AcceptConnect(earlier, 1_400); + Assert.That(server.CheckTimeout(1_600, 500), Is.True, "a stale Connect refreshed liveness"); + } + + /// + /// An accepted peer that never speaks again times out: accepting a Connect is the first thing + /// heard from it. + /// + [Test] + public void AnAcceptedPeerThatGoesSilentTimesOut() + { + var server = Session(500); + server.AcceptConnect(Session(10).CreateConnect(0), 0); + Assert.That(server.CheckTimeout(100_000, 1_000), Is.True); + } + + [Test] + public void StaleConnectsAreRecognisedBySerialArithmetic() + { + var server = Session(500); + server.AcceptConnect(Session(3).CreateConnect(0), 0); + CultNetRudpPacket Connect(uint sequence) => Session(sequence).CreateConnect(0); + Assert.That(server.ConnectRepeats(Connect(uint.MaxValue - 1)), Is.True, "just before the wrap"); + Assert.That(server.ConnectRepeats(Connect(4)), Is.False, "just after"); + Assert.That(server.ConnectRepeats(Connect(3 + 4_096)), Is.False, "far after"); + Assert.That(server.ConnectRepeats(Connect(unchecked(3u - 4_095))), Is.True, "the window's edge"); + Assert.That(server.ConnectRepeats(Connect(unchecked(3u - 4_096))), Is.False, "past the window"); + } + + /// + /// A restarted client whose random first sequence lands just before the server's current + /// Connect is answered with an Ack, never an Accept. It does not retransmit that Connect for + /// ever: once the attempt times out it starts a fresh one with a newly drawn sequence, and is + /// admitted. + /// + [Test] + public void ARestartedClientWhoseFirstSequenceIsStaleConnectsAfterRedrawing() + { + var server = Session(500); + var old = Session(1_000); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + + var restarted = Session(900); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + // The old client has not acknowledged its Accept yet, so that is the reply: it names the + // old Connect, not this one. + var early = server.AcceptConnect(first, 1); + Assert.That(early.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(early, 1); + Assert.That(restarted.Connected, Is.False, "an Accept for another Connect connected the client"); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 1); + var reply = server.AcceptConnect(first, 2); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + restarted.Receive(reply, 2); + Assert.That(restarted.Connected, Is.False); + + var retransmitted = restarted.DueResends(1_000); + Assert.That(retransmitted, Has.Count.EqualTo(1)); + Assert.That(retransmitted[0].Sequence, Is.EqualTo(first.Sequence), "the attempt is still young"); + + var fresh = restarted.DueResends(3_500); + Assert.That(fresh, Has.Count.EqualTo(1)); + Assert.That(fresh[0].PacketType, Is.EqualTo(CultNetRudpPacketType.Connect)); + Assert.That(fresh[0].Sequence, Is.Not.EqualTo(first.Sequence), "the same Connect was retransmitted for ever"); + Assert.That(Encoding.UTF8.GetString(fresh[0].Payload), Is.EqualTo("join"), "the fresh attempt lost the Connect payload"); + + var accept = server.AcceptConnect(fresh[0], 3_500); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 3_500); + Assert.That(restarted.Connected, Is.True); + Assert.That(Names(server.Receive(Send(restarted, "hello"), 3_501)), Is.EqualTo(new[] { "hello" })); + } + + [Test] + public void AnAnsweredConnectIsNeverReplacedByTheAttemptTimeout() + { + var client = Session(10); + var server = Session(500); + var (connect, _) = Handshake(client, server); + var resent = client.DueResends(60_000); + Assert.That(resent.All(p => p.PacketType != CultNetRudpPacketType.Connect || p.Sequence == connect.Sequence), Is.True, "a connected client started a new attempt"); + Assert.That(client.Connected, Is.True); + } + + /// + /// A pinned client that restarts on a new port against a server-mode transport is a new client: + /// the Connect starts a new generation and the client is admitted, though its sequence is the + /// one the old client's Connect carried. + /// + [Test] + public void ServerModeAdmitsAPinnedClientThatRestartsOnANewPort() + { + using var serverSocket = Bind(); + using var firstSocket = Bind(); + using var secondSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(Socket from, CultNetRudpPacket packet) => from.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(1); + ToServer(firstSocket, first.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + first.Receive(Drain(firstSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(firstSocket, Send(first, "hello")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("hello")); + + var second = Session(1); + ToServer(secondSocket, second.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + var accepts = Drain(secondSocket).Where(p => p.PacketType == CultNetRudpPacketType.Accept).ToArray(); + Assert.That(accepts, Is.Not.Empty, "the restarted client on a new port was not admitted"); + second.Receive(accepts[0], 0); + ToServer(secondSocket, Send(second, "after")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("after")); + } + + // Ack Cut 1d, batch 2: the connect-attempt timeout. + + /// + /// A server that keeps a session answers a client's repeated or stale Connect with an Ack that + /// may name the client's pending Connect. That Ack says no session started: only an Accept the + /// client honours retires its Connect, so the attempt still times out and is replaced. + /// + [Test] + public void AnAckNamingThePendingConnectDoesNotRetireIt() + { + var server = Session(500); + var old = Session(1); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 0); + + var restarted = Session(1); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + var reply = server.AcceptConnect(first, 1); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + Assert.That(NamesSequence(reply, first.Sequence), Is.True, "the Ack must name the pending Connect for this test to bite"); + restarted.Receive(reply, 1); + Assert.That(restarted.Connected, Is.False); + Assert.That(restarted.PendingReliableSequences, Is.EqualTo(new[] { first.Sequence }), "an Ack retired the Connect"); + + var fresh = restarted.DueResends(3_000); + Assert.That(fresh, Has.Count.EqualTo(1), "the client waits for ever with nothing to resend"); + Assert.That(Encoding.UTF8.GetString(fresh[0].Payload), Is.EqualTo("join")); + var accept = server.AcceptConnect(fresh[0], 3_000); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 3_000); + Assert.That(restarted.Connected, Is.True); + } + + [Test] + public void ALateCopyOfAnAbandonedConnectIsStaleOnceItsReplacementIsAccepted() + { + var server = Session(500); + var client = Session(10); + var abandoned = client.CreateConnect(0); + var fresh = client.DueResends(3_000); + Assert.That(fresh, Has.Count.EqualTo(1)); + Assert.That(fresh[0].Sequence, Is.EqualTo(abandoned.Sequence + 4_095)); + var accept = server.AcceptConnect(fresh[0], 3_001); + client.Receive(accept, 3_002); + server.Receive(client.CreateAckForReceived(accept.Sequence), 3_002); + Assert.That(client.Connected, Is.True); + var a = Send(client, "a"); + Assert.That(Names(server.Receive(a, 3_003)), Is.EqualTo(new[] { "a" })); + client.Receive(server.CreateAckForReceived(a.Sequence), 3_003); + + Assert.That(server.ConnectRepeats(abandoned), Is.True, "the late copy would restart the server"); + var reply = server.AcceptConnect(abandoned, 3_004); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + client.Receive(reply, 3_005); + Assert.That(client.Connected, Is.True); + Assert.That(Names(server.Receive(Send(client, "b"), 3_006)), Is.EqualTo(new[] { "b" })); + } + + [Test] + public void AServerSittingAtTheJumpIsLeftByTheNextAttempt() + { + var server = Session(500); + var old = Session(5_095); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 0); + + var restarted = Session(1_000); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + restarted.Receive(server.AcceptConnect(first, 1), 1); + Assert.That(restarted.Connected, Is.False); + + var second = restarted.DueResends(3_000)[0]; + Assert.That(second.Sequence, Is.EqualTo(5_095u), "the jump lands on the server's generation"); + var reply = server.AcceptConnect(second, 3_001); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + restarted.Receive(reply, 3_001); + Assert.That(restarted.Connected, Is.False); + + var third = restarted.DueResends(6_000)[0]; + Assert.That(third.Sequence, Is.EqualTo(5_095u + 4_095u)); + var accept = server.AcceptConnect(third, 6_001); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 6_001); + Assert.That(restarted.Connected, Is.True); + } + + /// + /// A pinned client that restarts on the same address is a repeat by sequence, so the listener + /// answers with an Ack; the client's attempt times out, and the fresh Connect (another + /// sequence) starts a new generation the listener admits. + /// + [Test] + public void TheListenerAdmitsAPinnedClientThatRestartsOnTheSameAddress() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(CultNetRudpPacket packet) => peerSocket.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(1); + ToServer(first.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + var accept = Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept); + first.Receive(accept, 0); + ToServer(first.CreateAckForReceived(accept.Sequence)); + server.ReceiveOnce(); + Drain(peerSocket); + + var restarted = Session(1); + ToServer(restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + foreach (var packet in Drain(peerSocket)) + restarted.Receive(packet, 1); + Assert.That(restarted.Connected, Is.False); + + ToServer(restarted.DueResends(3_000)[0]); + server.ReceiveOnce(); + foreach (var packet in Drain(peerSocket)) + restarted.Receive(packet, 3_001); + Assert.That(restarted.Connected, Is.True, "the restarted pinned client was never admitted"); + } + + // Transports. + + [Test] + public void ServerModeAdmitsARestartedClientKeepsItsSessionOnARetransmitAndAdmitsANewEndpoint() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var otherSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(Socket from, CultNetRudpPacket packet) => from.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(50_000); + var connect = first.CreateConnect(0, Encoding.UTF8.GetBytes("same")); + ToServer(peerSocket, connect); + server.ReceiveOnce(); + first.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + var sent = new[] { "c0", "c1", "c2" }.Select(name => Send(first, name)).ToArray(); + foreach (var packet in sent) + { + ToServer(peerSocket, packet); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo(Encoding.UTF8.GetString(packet.Payload))); + } + + // A retransmitted Connect is a repeat: what was delivered stays delivered once. + ToServer(peerSocket, connect); + server.ReceiveOnce(); + ToServer(peerSocket, sent[0]); + Assert.That(server.ReceiveOnce(), Is.Null, "the retransmitted Connect reset the session"); + + // The same client restarted on the same address, connection id and payload. + Drain(peerSocket); + var second = Session(7); + ToServer(peerSocket, second.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + second.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(peerSocket, Send(second, "after restart")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("after restart")); + + // A Connect from a new endpoint claims the transport. + var third = Session(9); + ToServer(otherSocket, third.CreateConnect(0, Encoding.UTF8.GetBytes("elsewhere"))); + server.ReceiveOnce(); + third.Receive(Drain(otherSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(otherSocket, Send(third, "from elsewhere")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("from elsewhere")); + } + + [Test] + public void TheListenerAdmitsARestartedClientWithTheSameAddressIdAndPayload() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var ended = new List(); + server.PeerDisconnected += ended.Add; + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(CultNetRudpPacket packet) => peerSocket.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(50_000); + ToServer(first.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + first.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + for (var index = 0; index < 3; index++) + { + ToServer(Send(first, $"c{index}")); + Assert.That(server.ReceiveOnce(), Is.Not.Null); + } + Assert.That(ended, Is.Empty); + + var second = Session(7); + ToServer(second.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + Assert.That(ended, Has.Count.EqualTo(1), "the restarted client was taken for a retransmit"); + second.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(Send(second, "after restart")); + var frame = server.ReceiveOnce(); + Assert.That(frame, Is.Not.Null); + Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("after restart")); + } + + /// + /// Windows reports the ICMP port-unreachable for a datagram sent to a closed port as a connection + /// reset on the next receive. Nothing ended, so nothing may be reported. Other platforms do not + /// raise it, so the test is inconclusive there. + /// + [Test] + public void AnIcmpConnectionResetDoesNotEndTheSession() + { + Assume.That(OperatingSystem.IsWindows(), "only Windows raises an ICMP connection reset on UDP"); + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(Session(1).CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Connected, Is.True); + + var closed = Bind(); + var closedEndPoint = closed.LocalEndPoint!; + closed.Dispose(); + serverSocket.SendTo(new byte[] { 1 }, closedEndPoint); + System.Threading.Thread.Sleep(50); + server.ReceiveOnce(); + Assert.That(server.DisconnectReason, Is.Null, "an ICMP reset claimed an ending that did not happen"); + Assert.That(server.Connected, Is.True); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs new file mode 100644 index 00000000..aeca5bc4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs @@ -0,0 +1,134 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Text; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// Ordered delivery has one owner: the contiguous received-through watermark. A frame is held for + /// exactly one reason, a reliable sequence below it has not arrived, and it is delivered in the call + /// that fills the last such gap, whichever channel the gap belonged to. + /// + public sealed class CultNetRudpOrderedDeliveryTests + { + private static (CultNetRudpSession Client, CultNetRudpSession Server) Handshake() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 500 }); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + client.Receive(accept, 0); + return (client, server); + } + + private static CultNetRudpSendOptions Reliable(bool ordered) => new CultNetRudpSendOptions { Reliable = true, Ordered = ordered }; + + private static CultNetRudpPacket Send(CultNetRudpSession session, string channel, string payload, bool ordered) + => session.Send(channel, Encoding.UTF8.GetBytes(payload), Reliable(ordered)); + + private static string[] Names(CultNetRudpReceiveResult result) + => result.Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(); + + [Test] + public void OrderedFrameWaitsForAGapFilledByAnotherChannel() + { + var (sender, receiver) = Handshake(); + var o1 = Send(sender, "schema", "o1", true); + var u = Send(sender, "rel", "u", false); + var o2 = Send(sender, "schema", "o2", true); + var o3 = Send(sender, "schema", "o3", true); + var o4 = Send(sender, "schema", "o4", true); + + Assert.That(Names(receiver.Receive(o1, 1)), Is.EqualTo(new[] { "o1" })); + Assert.That(Names(receiver.Receive(o2, 2)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(o3, 3)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(o4, 4)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(u, 5)), Is.EqualTo(new[] { "u", "o2", "o3", "o4" })); + } + + [Test] + public void TwoOrderedChannelsReleaseEachOtherInSequenceOrder() + { + var (sender, receiver) = Handshake(); + var a1 = Send(sender, "schema", "A1", true); + var b1 = Send(sender, "other", "B1", true); + var a2 = Send(sender, "schema", "A2", true); + var a3 = Send(sender, "schema", "A3", true); + + var delivered = new List(); + delivered.AddRange(Names(receiver.Receive(a1, 1))); + delivered.AddRange(Names(receiver.Receive(a2, 2))); + delivered.AddRange(Names(receiver.Receive(a3, 3))); + Assert.That(delivered, Is.EqualTo(new[] { "A1" }), "B1 is missing, so A2 and A3 wait"); + Assert.That(Names(receiver.Receive(b1, 4)), Is.EqualTo(new[] { "B1", "A2", "A3" })); + } + + [Test] + public void ChannelFirstUsedAfterOtherTrafficLosesNothing() + { + var (sender, receiver) = Handshake(); + var c1 = Send(sender, "late", "C1", true); + var c2 = Send(sender, "late", "C2", true); + var x = Send(sender, "schema", "X", true); + + Assert.That(Names(receiver.Receive(x, 1)), Is.Empty, "C1 is missing"); + Assert.That(Names(receiver.Receive(c2, 2)), Is.Empty, "C1 is missing"); + Assert.That(Names(receiver.Receive(c1, 3)), Is.EqualTo(new[] { "C1", "C2", "X" })); + } + + [Test] + public void AcceptSeedsTheWatermarkOfTheConnectingSide() + { + var (client, server) = Handshake(); + var s1 = Send(server, "schema", "s1", true); + var s2 = Send(server, "schema", "s2", true); + + Assert.That(Names(client.Receive(s2, 1)), Is.Empty); + Assert.That(Names(client.Receive(s1, 2)), Is.EqualTo(new[] { "s1", "s2" })); + } + + [Test] + public void ConnectSeedsTheWatermarkOfTheAcceptingSide() + { + var (client, server) = Handshake(); + var first = Send(client, "schema", "first", true); + var second = Send(client, "schema", "second", true); + + Assert.That(Names(server.Receive(second, 1)), Is.Empty); + Assert.That(Names(server.Receive(first, 2)), Is.EqualTo(new[] { "first", "second" })); + } + + [Test] + public void DuplicateOfAHeldFrameIsNotDeliveredTwice() + { + var (sender, receiver) = Handshake(); + var s1 = Send(sender, "schema", "s1", true); + var s2 = Send(sender, "schema", "s2", true); + + Assert.That(Names(receiver.Receive(s2, 1)), Is.Empty); + Assert.That(Names(receiver.Receive(s2, 2)), Is.Empty); + Assert.That(Names(receiver.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" })); + Assert.That(Names(receiver.Receive(s2, 4)), Is.Empty); + } + + [Test] + public void ResetPeerStateForgetsHeldFramesAndReseedsTheWatermarkFromTheNextHandshake() + { + var (sender, receiver) = Handshake(); + Send(sender, "schema", "s1", true); + var s2 = Send(sender, "schema", "s2", true); + Assert.That(Names(receiver.Receive(s2, 1)), Is.Empty); + + receiver.ResetPeerState(); + var next = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 100 }); + next.Receive(receiver.AcceptConnect(next.CreateConnect(0), 2), 2); + var d1 = Send(next, "schema", "d1", true); + var d2 = Send(next, "schema", "d2", true); + + // The held s2 died with the reset, and the watermark follows the new peer. + Assert.That(Names(receiver.Receive(d2, 3)), Is.Empty); + Assert.That(Names(receiver.Receive(d1, 4)), Is.EqualTo(new[] { "d1", "d2" })); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs new file mode 100644 index 00000000..35315bc4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs @@ -0,0 +1,215 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A write belongs to the session generation it was issued in. Every way a generation ends drops + /// the writes it still owed, and a flush belongs to the generation it started in. + /// + public sealed class CultNetRudpSessionEndingTests + { + private const uint ConnectionId = 0x10203070; + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + return socket; + } + + private static (CultNetRudpSession Client, CultNetRudpSession Server) ClientWithALostWrite() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + client.Receive(server.AcceptConnect(client.CreateConnect(0), 0), 0); + client.Send("schema", Encoding.UTF8.GetBytes("owed to the old session"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + // A fragmented write larger than the send window leaves part of it queued. + client.SendMany("schema", new byte[40 * 8], new CultNetRudpSendOptions { Reliable = true, Ordered = true }, maxFragmentBytes: 8); + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(41)); + Assert.That(client.QueuedReliablePacketCount, Is.GreaterThan(0)); + return (client, server); + } + + [TestCase("peer Disconnect")] + [TestCase("local disconnect")] + [TestCase("timeout")] + [TestCase("reset")] + public void EveryWayASessionEndsDropsTheWritesItOwed(string ending) + { + var (client, server) = ClientWithALostWrite(); + switch (ending) + { + case "peer Disconnect": client.Receive(server.CreateDisconnect(Encoding.UTF8.GetBytes("bye")), 1); break; + case "local disconnect": client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")); break; + case "timeout": Assert.That(client.CheckTimeout(1_000, 10), Is.True); break; + case "reset": client.ResetPeerState(); break; + default: throw new ArgumentException(ending); + } + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(0), "the write survived the end"); + + var nextServer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 900 }); + nextServer.AcceptConnect(client.CreateConnect(2_000), 2_000); + foreach (var resend in client.DueResends(60_000)) + { + Assert.That( + nextServer.Receive(resend, 60_000).Delivered, + Is.Empty, + "the old session's write was delivered in the next session"); + } + } + + private static (CultNetRudpSocketTransportConnection Client, Socket Peer, EndPoint ClientEndPoint) ConnectedClient() + { + var peerSocket = Bind(); + var client = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = Bind(), + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = peerSocket.LocalEndPoint!, + ConnectionId = ConnectionId + }); + client.Connect("hello"); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + var received = peerSocket.ReceiveFrom(buffer, ref from); + var peer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + var accept = peer.AcceptConnect(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray()), 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(accept), from); + client.ReceiveOnce(); + Assert.That(client.Connected, Is.True); + return (client, peerSocket, from); + } + + [Test] + public void AFlushFailsAfterALocalDisconnectOrATimeout() + { + var (client, peerSocket, _) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed")); + client.Disconnect(); + Assert.Throws(() => client.FlushReliable(TimeSpan.FromMilliseconds(200))); + } + + var (timedOut, secondPeer, _) = ConnectedClient(); + using (timedOut) + using (secondPeer) + { + timedOut.Send("schema", Encoding.UTF8.GetBytes("owed")); + Thread.Sleep(5); + Assert.That(timedOut.CheckTimeout(1), Is.True); + Assert.Throws(() => timedOut.FlushReliable(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void AFlushThatStartedBeforeTheEndFailsEvenAfterALaterConnect() + { + var (client, peerSocket, _) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed")); + var flush = Task.Run(() => client.FlushReliable(TimeSpan.FromSeconds(5))); + Thread.Sleep(50); + client.Disconnect(); + client.Connect("hello again"); + + var error = Assert.Throws(() => flush.Wait(TimeSpan.FromSeconds(10)))!; + Assert.That(error.InnerException, Is.InstanceOf()); + } + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + [Test] + public void AFlushStartedAfterAReconnectWaitsOnTheNewSession() + { + var (client, peerSocket, clientEndPoint) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed to the old session")); + client.Disconnect(); + client.Connect("hello again"); + var connect = Drain(peerSocket).Last(packet => packet.PacketType == CultNetRudpPacketType.Connect); + var accept = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 700 }).AcceptConnect(connect, 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(accept), clientEndPoint); + client.ReceiveOnce(); + + Assert.DoesNotThrow(() => client.FlushReliable(TimeSpan.FromMilliseconds(500))); + } + } + + [Test] + public void AFlushStartedAfterTheServerAcceptsANewConnectWaitsOnTheNewSession() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var first = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(first.CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(first.CreateDisconnect()), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + + var next = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(next.CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + var accept = Drain(peerSocket).Last(packet => packet.PacketType == CultNetRudpPacketType.Accept); + next.Receive(accept, 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(next.CreateAck()), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + + Assert.DoesNotThrow(() => server.FlushReliable(TimeSpan.FromMilliseconds(500))); + } + + [Test] + public void AnEndingDoesNotForgetWhatWasReceivedFromThePeer() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + client.Receive(server.AcceptConnect(client.CreateConnect(0), 0), 0); + var ordered = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + var s1 = client.Send("schema", Encoding.UTF8.GetBytes("s1"), ordered); + var s2 = client.Send("schema", Encoding.UTF8.GetBytes("s2"), ordered); + Assert.That(server.Receive(s2, 1).Delivered, Is.Empty, "s1 is missing, so s2 is held"); + + Assert.That(server.CheckTimeout(1_000, 10), Is.True); + Assert.That( + server.Receive(s1, 1_001).Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(), + Is.EqualTo(new[] { "s1", "s2" }), + "the ending forgot the held frame"); + Assert.That(server.Receive(s2, 1_002).Delivered, Is.Empty, "the ending forgot what was received"); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index b5f4f08f..3aa2705d 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -195,6 +195,17 @@ namespace GameCult.Networking.Tests AssertErrorMatchesSchema(CultNetErrorMessage.ForReferenceOutsideTarget(ex)); } + [Test] + public void ErrorMessage_UnownedSchemaRealBytes_MatchesItsSchema() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + + Assert.That(message.Code, Is.EqualTo("unowned_schema")); + Assert.That(message.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(message.Details.Value, Is.EqualTo("some.schema.v1")); + AssertErrorMatchesSchema(message); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index e32ca91a..8d01a7c3 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -730,26 +730,25 @@ namespace GameCult.Networking.Tests InitialSequence = 200, ResendDelayMs = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 7, Sequence = 1, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 7, Sequence = 2, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var second = sender.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var third = sender.Send("schema", Encoding.UTF8.GetBytes("third"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); - Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 10, 11, 12 })); + Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 11, 12, 13 })); receiver.Receive(first); receiver.Receive(third); var ackWithGap = receiver.CreateAck(); - Assert.That(ackWithGap.Ack, Is.EqualTo(12)); - Assert.That(ackWithGap.AckMask, Is.EqualTo(0b10u | (1u << 9))); + Assert.That(ackWithGap.Ack, Is.EqualTo(13)); + Assert.That(ackWithGap.AckMask, Is.EqualTo(0b110u)); sender.Receive(ackWithGap); - Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 11 })); + Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 12 })); receiver.Receive(second); var fullAck = receiver.CreateAck(); - Assert.That(fullAck.Ack, Is.EqualTo(12)); - Assert.That(fullAck.AckMask, Is.EqualTo(0b11u | (1u << 9))); + Assert.That(fullAck.Ack, Is.EqualTo(13)); + Assert.That(fullAck.AckMask, Is.EqualTo(0b111u)); sender.Receive(fullAck); Assert.That(sender.PendingReliableSequences, Is.Empty); } @@ -763,7 +762,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, ResendDelayMs = 100 }); - session.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 99, Sequence = 50, ChannelId = "control" }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 99, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var sent = session.Send("schema", Encoding.UTF8.GetBytes("payload"), new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 10 }); Assert.That(session.DueResends(90), Is.Empty); @@ -788,17 +787,14 @@ namespace GameCult.Networking.Tests ConnectionId = 991, InitialSequence = 500 }); - sender.Receive(new CultNetRudpPacket - { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 991, Sequence = 400, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket - { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 991, Sequence = 0, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var oldPacket = sender.Send( "snapshot", new byte[] { 1 }, new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 0 }); receiver.Receive(oldPacket); - foreach (var sequence in Enumerable.Range(2, 40).Select(value => (uint)value)) + foreach (var sequence in Enumerable.Range(3, 40).Select(value => (uint)value)) { receiver.Receive(new CultNetRudpPacket { @@ -813,16 +809,16 @@ namespace GameCult.Networking.Tests } sender.Receive(receiver.CreateAck()); - Assert.That(sender.PendingReliableSequences.First(), Is.EqualTo(1u)); + Assert.That(sender.PendingReliableSequences.First(), Is.EqualTo(2u)); - var oldRetransmit = sender.DueResends(10).Single(packet => packet.Sequence == 1); + var oldRetransmit = sender.DueResends(10).Single(packet => packet.Sequence == 2); receiver.Receive(oldRetransmit); var exactAck = receiver.CreateAckForReceived(oldRetransmit.Sequence); Assert.That(exactAck.Ack, Is.EqualTo(oldRetransmit.Sequence)); Assert.That(exactAck.AckMask, Is.Zero); sender.Receive(exactAck); - Assert.That(sender.PendingReliableSequences, Does.Not.Contain(1u)); + Assert.That(sender.PendingReliableSequences, Does.Not.Contain(2u)); } [Test] @@ -834,13 +830,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, ResendDelayMs = 1 }); - session.Receive(new CultNetRudpPacket - { - PacketType = CultNetRudpPacketType.Accept, - ConnectionId = 100, - Sequence = 5000, - ChannelId = "control" - }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 100, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var packets = Enumerable.Range(0, CultNetRudpSession.ReliableSendWindowPackets) .Select(index => session.Send("schema", Encoding.UTF8.GetBytes(index.ToString()), new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 0 })) @@ -909,7 +899,7 @@ namespace GameCult.Networking.Tests var accept = server.AcceptConnect(connect, 0); var repeat = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(connect)); repeat.Ack = accept.Sequence; - var reply = server.AnswerRepeatedConnect(repeat, 1); + var reply = server.AcceptConnect(repeat, 1); Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(0)); } @@ -968,13 +958,7 @@ namespace GameCult.Networking.Tests public void RudpSession_RejectsUnreliableOrderedDelivery() { var session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 196 }); - session.Receive(new CultNetRudpPacket - { - PacketType = CultNetRudpPacketType.Accept, - ConnectionId = 196, - Sequence = 50, - ChannelId = "control" - }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 196, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var error = Assert.Throws(() => session.Send( "schema", @@ -992,14 +976,14 @@ namespace GameCult.Networking.Tests InitialSequence = 1, MaxPendingReliablePackets = 2 }); - session.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 102, Sequence = 50, ChannelId = "control" }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 102, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); session.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); session.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var error = Assert.Throws(() => session.Send("schema", Encoding.UTF8.GetBytes("third"), new CultNetRudpSendOptions { Reliable = true, Ordered = true })); Assert.That(error!.Message, Does.Contain("reliable send queue is full")); - Assert.That(session.PendingReliableSequences, Is.EqualTo(new uint[] { 1, 2 })); + Assert.That(session.PendingReliableSequences, Is.EqualTo(new uint[] { 2, 3 })); var fragmented = new CultNetRudpSession(new CultNetRudpSessionOptions { @@ -1007,7 +991,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, MaxPendingReliablePackets = 3 }); - fragmented.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 103, Sequence = 50, ChannelId = "control" }); + fragmented.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 103, InitialSequence = 900 }).AcceptConnect(fragmented.CreateConnect(0), 0), 0); error = Assert.Throws(() => fragmented.SendMany( @@ -1032,8 +1016,8 @@ namespace GameCult.Networking.Tests ConnectionId = 123, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 123, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 123, Sequence = 91, ChannelId = "control" }); + var accept = receiver.AcceptConnect(sender.CreateConnect(0), 0); + sender.Receive(accept); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var second = sender.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); @@ -1062,8 +1046,7 @@ namespace GameCult.Networking.Tests ConnectionId = 124, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 124, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 124, Sequence = 91, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var control = sender.CreateAck(); @@ -1091,8 +1074,7 @@ namespace GameCult.Networking.Tests ConnectionId = 456, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 456, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 456, Sequence = 91, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var packets = sender.SendMany( "schema", @@ -1130,8 +1112,7 @@ namespace GameCult.Networking.Tests InitialSequence = 100, ResendDelayMs = 25 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = connectionId, Sequence = 0, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = connectionId, Sequence = 0, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var fragmentCount = CultNetRudpSession.ReliableSendWindowPackets + 17; var payload = Enumerable.Range(0, fragmentCount * 8).Select(index => (byte)(index % 251)).ToArray(); @@ -1269,6 +1250,16 @@ namespace GameCult.Networking.Tests Assert.That(Encoding.UTF8.GetString(ReceiveRudpFrame(client).Payload), Is.EqualTo("server still here")); } + /// Whether the packet's ack field and mask cover the sequence. + private static bool Acknowledges(CultNetRudpPacket packet, uint sequence) + { + if (packet.Ack == sequence) + return true; + return packet.Ack > sequence + && packet.Ack - sequence <= 32 + && (packet.AckMask & (1u << (int)(packet.Ack - sequence - 1))) != 0; + } + private static List DrainPackets(Socket socket) { var packets = new List(); @@ -1397,7 +1388,7 @@ namespace GameCult.Networking.Tests var goodbye = DrainPackets(peerSocket).Single(); Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); - Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + Assert.That(Acknowledges(goodbye, poison.Sequence), Is.False, "the goodbye acknowledged the refused frame"); // A retransmit is dropped, not acknowledged. peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); @@ -1444,7 +1435,7 @@ namespace GameCult.Networking.Tests var goodbye = DrainPackets(peerSocket).Single(); Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); - Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + Assert.That(Acknowledges(goodbye, poison.Sequence), Is.False, "the goodbye acknowledged the refused frame"); peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); Assert.That(server.ReceiveOnce(), Is.Null); @@ -1898,7 +1889,7 @@ namespace GameCult.Networking.Tests Assert.That(cache.Get(batched), Is.Null); }), Is.True); - Assert.That(published, Is.EqualTo(new[] { put })); + Assert.That(published, Is.EqualTo(new[] { put, batched }), "the batch publishes when the store commits it"); Assert.That(cache.Get(batched)?.Text, Is.EqualTo("batched")); var reopened = new CultCache(); reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); @@ -2627,7 +2618,8 @@ namespace GameCult.Networking.Tests var cache = new CultCache(); var delivering = new ManualResetEventSlim(); - cache.OnUpdate += (_, _) => delivering.Set(); + // Subscribed before the database, so it runs before the database's own observer of the same change. + using var loadDelivering = cache.Watch().Subscribe(_ => delivering.Set()); var database = new CultNetDatabase(cache); var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions { @@ -4211,13 +4203,13 @@ namespace GameCult.Networking.Tests Schema = "tests.networking_note.v1", Text = "two" }); - // Outside the shard's key prefix: must reach the non-shard snapshot but never the + // Outside the shard's key prefix (written to the cache directly: the database refuses a write no shard owns): must reach the non-shard snapshot but never the // shard-scoped one, so an unfiltered request still proves shard membership is enforced. - await database.PutAsync(new CultRecordKey("other-note:outside-shard"), new NetworkSchemaNote + await cache.UpsertAsync(new NetworkSchemaNote { Schema = "tests.networking_note.v1", Text = "outside" - }); + }, new CultRecordHandle(new CultRecordKey("other-note:outside-shard"))); var nonShard = registry.CreateRawSnapshotResponse(cache, "snapshot-nonshard-unfiltered", filter: null); var shardScoped = database.CreateShardSnapshotResponse(shard, "snapshot-shard-unfiltered", filter: null); @@ -5270,10 +5262,49 @@ namespace GameCult.Networking.Tests #pragma warning restore CS0618 } - [Test] - public void CultMeshRudpClient_DefaultBindAllowsRemoteRoutes() + private static string? RudpClientBoundHost(string endpoint, string? bindHost = null) { - Assert.That(new CultMeshRudpSocketOptions().BindHost, Is.EqualTo("0.0.0.0")); + using var client = CultMesh.CreateRudpClient( + "csharp-cultmesh-bind-client", + 0x10203060u, + endpoint, + new CultMeshRudpSocketOptions { BindHost = bindHost }); + return client.Profile.Transports[0].Host; + } + + // A client bound to loopback cannot send to another host on Windows, so a client for a + // remote endpoint binds the unspecified address of its family. + [Test] + public void CultMeshRudpClient_ForARemoteEndpoint_BindsTheUnspecifiedAddress() + { + Assert.That(RudpClientBoundHost("rudp://10.77.0.1:17872"), Is.EqualTo("0.0.0.0")); + Assert.That(RudpClientBoundHost("rudp://[2001:db8::1]:17872"), Is.EqualTo("::")); + } + + [Test] + public void CultMeshRudpClient_ForALoopbackEndpoint_BindsLoopback() + { + Assert.That(RudpClientBoundHost("rudp://127.0.0.1:17872"), Is.EqualTo("127.0.0.1")); + Assert.That(RudpClientBoundHost("rudp://[::1]:17872"), Is.EqualTo("::1")); + } + + [Test] + public void CultMeshRudpClient_BindsAnExplicitBindHostWhateverTheEndpoint() + { + Assert.That(RudpClientBoundHost("rudp://10.77.0.1:17872", "127.0.0.1"), Is.EqualTo("127.0.0.1")); + Assert.That(RudpClientBoundHost("rudp://127.0.0.1:17872", "0.0.0.0"), Is.EqualTo("0.0.0.0")); + } + + [Test] + public void CultMeshRudpServer_BindsEveryIPv4InterfaceUnlessToldOtherwise() + { + using var anywhere = CultMesh.CreateRudpServer("csharp-cultmesh-bind-server", 0x10203061u); + using var loopback = CultMesh.CreateRudpServer( + "csharp-cultmesh-bind-server", + 0x10203062u, + new CultMeshRudpSocketOptions { BindHost = "127.0.0.1" }); + Assert.That(anywhere.Profile.Transports[0].Host, Is.EqualTo("0.0.0.0")); + Assert.That(loopback.Profile.Transports[0].Host, Is.EqualTo("127.0.0.1")); } [Test] diff --git a/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs new file mode 100644 index 00000000..feb2cb3e --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4ProbeTests.cs @@ -0,0 +1,304 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass on Cut 4 (scratch; not for merge). Category Finding asserts the invariant Soul expects and is expected to + // fail on 13072fc5; Pin asserts a promise that should hold; Measure reports numbers. + public sealed class SoulP4ProbeTests + { + private const string ShardId = "soul-shard"; + private static readonly CultRecordKey One = new("soul:p4:one"); + private static readonly CultRecordKey Two = new("soul:p4:two"); + private static readonly CultRecordKey Three = new("soul:p4:three"); + + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + private sealed class Store : ICultNetShardMutationLogStore + { + private readonly object _gate = new(); + private readonly SortedDictionary _entries = new(); + private long _compacted; + public Func RefuseAppend { get; set; } = _ => false; + public bool RefuseCompact { get; set; } + public int AppendDelayMs { get; set; } + public ManualResetEventSlim InAppend { get; } = new(); + + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) + { + lock (_gate) + { + var e = _entries.Values.Where(x => x.Sequence > afterSequence); + return (limit.HasValue ? e.Take(limit.Value) : e).ToArray(); + } + } + + public void Append(string shardId, CultNetShardLogEntryMessage entry) + { + InAppend.Set(); + if (AppendDelayMs > 0) Thread.Sleep(AppendDelayMs); + lock (_gate) + { + if (RefuseAppend(entry.Sequence)) throw new IOException($"refused {entry.Sequence}"); + _entries[entry.Sequence] = entry; + } + } + + public long GetCompactedThrough(string shardId) { lock (_gate) return _compacted; } + + public void CompactThrough(string shardId, long sequence) + { + lock (_gate) + { + if (RefuseCompact) throw new IOException("refused compact"); + _compacted = Math.Max(_compacted, sequence); + foreach (var k in _entries.Keys.Where(k => k <= sequence).ToArray()) _entries.Remove(k); + } + } + } + + private sealed class Peer : ICultNetShardLogFetcher, ICultNetShardSnapshotFetcher, IDisposable + { + private readonly Server _server; + private readonly CultNetDatabaseServer _dbServer; + private readonly CultNetDatabase _db; + public Peer(CultCache cache, CultNetDatabase db) + { + _db = db; + _server = new Server(cache, ServerSecurityOptions.Development()); + _dbServer = new CultNetDatabaseServer(_server, db); + } + public CultNetShardLogResponseMessage Log(long after) => _dbServer.CreateShardLogResponse(new CultNetShardLogRequestMessage + { MessageId = "pull", ShardId = ShardId, ShardEpoch = 1, AfterSequence = after }); + public Task FetchAsync(CultNetShardDescriptor shard, long afterSequence, int? limit = null) => Task.FromResult(Log(afterSequence)); + public Task FetchAsync(CultNetShardDescriptor shard) => Task.FromResult(_db.CreateShardSnapshotResponse(_db.Shards[0], "snap")); + public void Dispose() { _dbServer.Dispose(); _server.Dispose(); } + } + + private static int StashCount(CultNetDatabase db) + { + var field = typeof(CultNetDatabase).GetField("_stash", BindingFlags.NonPublic | BindingFlags.Instance)!; + var stash = (System.Collections.ICollection)field.GetValue(db)!; + lock (typeof(CultNetDatabase).GetField("_logGate", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(db)!) + return stash.Count; + } + + // F1: the recorded residual is "tail reuse". A hole in the middle of the log after a double refusal and a restart + // strands a replica behind it for good: the restarted primary has no floor, the store serves S+1 after S-1. + [Test, Category("SoulFinding")] + public async Task F1_MidLogHoleAfterDoubleRefusalAndRestartStrandsAReplica() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + await primary.PutAsync(One, Note("one")); + using (var p0 = new Peer(cache, primary)) + using (var r0 = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = p0, SnapshotFetcher = p0 })) + await r0.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + store.RefuseCompact = false; + await primary.PutAsync(Three, Note("three")); + primary.Dispose(); + + var restarted = Db(cache, true, store); + using var peer = new Peer(cache, restarted); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + var errors = new List(); + for (var attempt = 0; attempt < 3; attempt++) + { + try { await replicator.PullOnceAsync(ShardId); } + catch (Exception e) { errors.Add(e.GetType().Name + ": " + e.Message); } + } + + TestContext.Out.WriteLine($"store={string.Join(",", store.Read(ShardId).Select(e => e.Sequence))} floor={restarted.GetCompactedMutationLogSequence(ShardId)} errors=[{string.Join(" | ", errors)}]"); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + // F2: Q-P5 A says the writer gets a typed CultNetShardLogException. A commit of two changes on a refusing store + // hands the writer an AggregateException; a writer's catch (CultNetShardLogException) misses it. + [Test, Category("SoulFinding")] + public void F2_ABatchOnARefusingStoreReachesTheWriterAsTheTypedException() + { + var store = new Store { RefuseAppend = _ => true }; + var cache = new CultCache(); + _ = Db(cache, true, store); + Exception? caught = null; + try + { + cache.Commit(batch => + { + batch.Upsert(Note("one"), new CultRecordHandle(One)); + batch.Upsert(Note("two"), new CultRecordHandle(Two)); + }); + } + catch (Exception e) { caught = e; } + TestContext.Out.WriteLine($"writer caught {caught?.GetType().FullName}"); + Assert.That(caught, Is.InstanceOf()); + } + + // F3: main publishes PutPredictedAsync as Predicted and records the prediction whatever instance it is given. The + // shape rule refuses a door whose instance is already the cached one, so a mutate-then-predict is published as + // Updated, is logged on a primary, and its reconciliation is published as Updated. + [Test, Category("SoulFinding")] + public async Task F3_APredictionOfTheCachedInstanceIsPublishedAsAPrediction() + { + var cache = new CultCache(); + var db = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var shard = db.Shards[0]; + var kinds = new List(); + db.Watch().Subscribe(c => kinds.Add(c.Kind)); + await db.PutPredictedAsync(One, Note("p1")); + var held = cache.Get(One)!; + held.Text = "p2"; + await db.PutPredictedAsync(One, held); + var auth = db.Documents.CreateRawDocumentPutMessage("a", new CultRecordHandle(One), Note("auth")); + auth.ShardId = shard.ShardId; + auth.ShardEpoch = shard.Epoch; + await db.ApplyPutAsync(auth); + TestContext.Out.WriteLine($"kinds=[{string.Join(",", kinds)}] log=[{string.Join(",", db.GetMutationLog(shard.ShardId).Select(e => e.Kind))}]"); + Assert.That(kinds, Is.EqualTo(new[] { CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Predicted, CultNetDatabaseChangeKind.Reconciled })); + Assert.That(db.GetMutationLog(shard.ShardId).Select(e => e.Kind), Is.EqualTo(new[] { CultNetDatabaseChangeKind.Updated })); + } + + // F4: the journal holds the cache gate across the log append, and cache reads take the gate, so a reader of an + // unrelated key waits for the whole append. + [Test, Category("SoulFinding"), Ignore("Accepted follow-up F4: a log append runs under the cache gate, so readers wait; documented on ICultNetShardMutationLogStore.")] + public async Task F4_AReaderOfAnotherKeyDoesNotWaitForALogAppend() + { + var store = new Store { AppendDelayMs = 400 }; + var cache = new CultCache(); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + var db = Db(cache, true, store); + var writer = Task.Run(() => cache.UpsertAsync(Note("one"), new CultRecordHandle(One)).GetAwaiter().GetResult()); + Assert.That(store.InAppend.Wait(5000), Is.True); + var clock = Stopwatch.StartNew(); + _ = cache.Get(Two); + var waited = clock.Elapsed.TotalMilliseconds; + await writer; + TestContext.Out.WriteLine($"reader of an unrelated key waited {waited:0.0} ms for a 400 ms append"); + Assert.That(waited, Is.LessThan(100)); + } + + // P1: the constructor registers the observer before the journal, so a change stashed by the journal always finds + // its publisher, even while another thread writes the cache during construction. + [Test, Category("SoulPin")] + public void P1_ConstructionUnderConcurrentWritesOrphansNoStash() + { + var orphans = 0; + for (var round = 0; round < 200; round++) + { + var cache = new CultCache(); + using var stop = new CancellationTokenSource(); + var writer = Task.Run(() => + { + var i = 0; + while (!stop.IsCancellationRequested) + cache.UpsertAsync(Note("w" + i++), new CultRecordHandle(new CultRecordKey("soul:p4:race:" + (i % 4)))).GetAwaiter().GetResult(); + }); + Thread.SpinWait(200 * (round % 7)); + var db = Db(cache, true, null); + Thread.SpinWait(2000); + stop.Cancel(); + writer.GetAwaiter().GetResult(); + orphans += StashCount(db); + db.Dispose(); + } + + Assert.That(orphans, Is.EqualTo(0), "no stash entry outlives its admission"); + } + + // P2: every stash entry is removed by its publication. + [Test, Category("SoulPin")] + public async Task P2_TheStashDrainsAfterEveryWrite() + { + var cache = new CultCache(); + var db = Db(cache, true, new Store()); + await db.PutAsync(One, Note("one")); + await cache.UpsertAsync(Note("two"), new CultRecordHandle(Two)); + cache.Commit(batch => batch.Upsert(Note("three"), new CultRecordHandle(Three))); + await db.DeleteAsync(One); + Assert.That(StashCount(db), Is.EqualTo(0)); + } + + // P3: a replica never applies an entry past a hole. The primary burns 2 with compaction refused, then logs 3; a + // replica at 1 is answered compacted_history and resyncs, never applying 3 on top of 1. + [Test, Category("SoulPin")] + public async Task P3_AReplicaAtTheHoleResyncsRatherThanSkippingIt() + { + var store = new Store { RefuseAppend = s => s == 2, RefuseCompact = true }; + var cache = new CultCache(); + var primary = Db(cache, true, store); + using var peer = new Peer(cache, primary); + var replicaCache = new CultCache(); + var replica = Db(replicaCache, false, null); + using var replicator = new CultNetShardReplicator(replica, new CultNetShardReplicatorOptions { Fetcher = peer, SnapshotFetcher = peer }); + await primary.PutAsync(One, Note("one")); + await replicator.PullOnceAsync(ShardId); + try { await primary.PutAsync(Two, Note("two")); } catch (CultNetShardLogException) { } + await primary.PutAsync(Three, Note("three")); + Assert.That(peer.Log(1).Reason, Is.EqualTo("compacted_history")); + var applied = await replicator.PullOnceAsync(ShardId); + Assert.That(applied, Is.EqualTo(3)); + Assert.That(new[] { One, Two, Three }.Select(k => replicaCache.Get(k)?.Text), Is.EqualTo(new[] { "one", "two", "three" })); + } + + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion; writes a log file.")] + public async Task M1_BareWriteLatencyWithTheFileLogStore() + { + var root = Path.Combine(Path.GetTempPath(), "soulp4-log-" + Guid.NewGuid().ToString("N")); + try + { + var cache = new CultCache(); + _ = Db(cache, true, new CultNetFileShardMutationLogStore(root)); + var times = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await cache.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + times.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"file store bare write ms: first100 avg {times.Take(100).Average():0.000}, 400-500 avg {times.Skip(400).Take(100).Average():0.000}, last100 avg {times.Skip(900).Average():0.000}, max {times.Max():0.0}"); + var bare = new CultCache(); + var t2 = new List(); + for (var i = 0; i < 1000; i++) + { + var clock = Stopwatch.StartNew(); + await bare.UpsertAsync(Note("n" + i), new CultRecordHandle(new CultRecordKey("soul:p4:lat:" + i))); + t2.Add(clock.Elapsed.TotalMilliseconds); + } + + TestContext.Out.WriteLine($"no database bare write ms: last100 avg {t2.Skip(900).Average():0.000}"); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, true); + } + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs new file mode 100644 index 00000000..a4796e7d --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4cProbeTests.cs @@ -0,0 +1,142 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4c probes. They guard: a committed put whose shard log failed is not answered as a refusal (Q1); reads and watch agree + // on what a database serves (Q2); a replica log apply leaves out a row its shard does not own (Q3); the unowned refusal is not a + // shard-authority exception (Q4); two databases over one cache aggregate their log failures (Q5). + public sealed class SoulP4cProbeTests + { + private const string ShardId = "soulc-shard"; + private static readonly CultRecordKey One = new("soulc:one"); + private static readonly CultRecordKey Unowned = new("soulc:unowned"); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private sealed class RefusingStore : ICultNetShardMutationLogStore + { + public IReadOnlyList Read(string shardId, long afterSequence = 0, int? limit = null) => Array.Empty(); + public void Append(string shardId, CultNetShardLogEntryMessage entry) => throw new IOException("refused " + entry.Sequence); + public long GetCompactedThrough(string shardId) => 0; + public void CompactThrough(string shardId, long sequence) { } + } + + private sealed class Logger : GameCult.Logging.ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static CultNetDatabase Db(CultCache cache, bool primary, ICultNetShardMutationLogStore? store) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [SchemaId(cache)])], + MutationLogStore = store + }); + + // Q1: the server's own remote-put path is a writer. A put that committed and could not be logged is answered as a + // failure: the peer is told its write failed, while the cache holds it and it was published. + [Test, Category("SoulFinding")] + public async Task Q1_ServerAnswersACommittedButUnloggedRemotePutAsAFailure() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, new RefusingStore()); + using var server = new Server(cache, ServerSecurityOptions.Development()); + var logger = new Logger(); + server.Logger = logger; + using var databaseServer = new CultNetDatabaseServer(server, database); + var handler = (Func)typeof(CultNetDatabaseServer) + .GetField("_putHandler", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(databaseServer)!; + var peer = (CultNetServerPeer)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + var message = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(One), Note("wire")); + message.ShardId = ShardId; + message.ShardEpoch = 1; + + try { await handler(message, peer); } catch (Exception) { } + + TestContext.WriteLine("errors: " + string.Join(" | ", logger.Errors)); + Assert.That(cache.Get(One)?.Text, Is.EqualTo("wire"), "the put committed"); + Assert.That(logger.Errors, Has.None.Contains("raw put failed"), "a committed put must not be reported to the peer as a failed one"); + } + + // Q2: reads serve a cached row no shard owns, the database watch never publishes its changes: a view built from + // GetAll + Watch (CultMesh Collection, the subscription server's snapshot + WatchAllChanges) goes stale. + [Test, Category("SoulFinding")] + public async Task Q2_ReadsServeAnUnownedRowWhoseChangesTheWatchNeverPublishes() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var seen = new List(); + database.Watch().Subscribe(change => seen.Add(change.Document?.Body ?? "")); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v1" }, new CultRecordHandle(Unowned)); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "u", Body = "v2" }, new CultRecordHandle(Unowned)); + + var read = database.GetAll().Select(note => note.Body).ToArray(); + TestContext.WriteLine($"GetAll={string.Join(",", read)} watch={string.Join(",", seen)}"); + Assert.That(read.Length == 0 || seen.Count > 0, Is.True, "a row the database reads out must be one whose changes it publishes"); + } + + // Q3: a replica's log apply door writes a row its shard does not own. The snapshot apply path filters by + // ShardMatchesRawDocument; the log path does not. + [Test, Category("SoulFinding")] + public async Task Q3_AReplicaLogApplyWritesARowItsShardDoesNotOwn() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, store: null); + var put = replica.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "r", Body = "replicated" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + Exception? thrown = null; + try + { + await replica.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = DateTimeOffset.UtcNow.ToString("O"), ChangeKind = "added", Put = put }] + }); + } + catch (Exception exception) { thrown = exception; } + + TestContext.WriteLine($"thrown={thrown?.GetType().Name}: {thrown?.Message}; cached={cache.Get(Unowned)?.Body}; log={replica.GetMutationLog(ShardId).Count}"); + Assert.That(cache.Get(Unowned), Is.Null, "no shard owns it here, so no path writes it"); + } + + // Q4 (pin): the unowned refusal is not a shard-authority exception, so the server's routing path does not see it. + [Test, Category("SoulPin")] + public void Q4_TheUnownedRefusalIsNotAShardAuthorityException() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, store: null); + var put = database.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(Unowned), new MeshQuickstartNote { NoteId = "w", Body = "w" }); + var refusal = Assert.ThrowsAsync(async () => await database.ApplyPutAsync(put))!; + Assert.That(refusal, Is.Not.InstanceOf()); + } + + // Q5 (pin): two databases over one cache, both refusing: a writer's catch (CultNetShardLogException) misses. + [Test, Category("SoulPin")] + public void Q5_TwoDatabasesOnOneCacheAggregateTheirLogFailures() + { + var cache = new CultCache(); + _ = Db(cache, primary: true, new RefusingStore()); + _ = Db(cache, primary: true, new RefusingStore()); + var failure = Assert.Throws(() => cache.Commit(batch => + batch.Upsert(Note("x"), new CultRecordHandle(One))))!; + Assert.That(failure.InnerExceptions.OfType().Count(), Is.EqualTo(2)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs new file mode 100644 index 00000000..6623f8a1 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulP4dProbeTests.cs @@ -0,0 +1,201 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // Soul pass 4d probes. They guard the replica log relay (a skipped entry is relayed as the primary sent it, in memory and on + // disk), key-prefixed shard reads, and measure two costs; the measurements are explicit. + public sealed class SoulP4dProbeTests + { + private const string ShardId = "soupd-shard"; + private static readonly string Now = DateTimeOffset.UtcNow.ToString("O"); + + private static string NoteSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + private static string MeshSchema(CultCache cache) => cache.Registry.GetRequired().SchemaId; + + private static CultNetDatabase Db(CultCache cache, bool primary, string[] schemas, string? keyPrefix = null, ICultNetShardMutationLogStore? store = null) => + new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: schemas, keyPrefix: keyPrefix)], + MutationLogStore = store + }); + + private static CultNetShardLogResponseMessage UnownedPutLog(CultNetDatabase anyDb, string key) + { + var put = anyDb.Documents.CreateRawDocumentPutMessage("wire", new CultRecordHandle(new CultRecordKey(key)), new MeshQuickstartNote { NoteId = "p", Body = "primary-put" }); + put.ShardId = ShardId; + put.ShardEpoch = 1; + return new CultNetShardLogResponseMessage + { + MessageId = "log", + ShardId = ShardId, + ShardEpoch = 1, + Entries = [new CultNetShardLogEntryMessage { Sequence = 1, CommittedAt = Now, ChangeKind = "added", Put = put }] + }; + } + + // Q3a: a replica (no store) that skips an unowned put relays it to a chained replica as a delete. + [Test, Category("SoulFinding")] + public async Task Q3a_MemoryLogRelaysASkippedPutAsARemoval() + { + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)]); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + TestContext.Out.WriteLine($"memory relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null} delete={relayed[0].Delete != null}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added"), "the relayed entry keeps the primary's kind"); + } + + // Q3b: the same replica with a durable store relays the primary's put (disk and memory disagree). + [Test, Category("SoulFinding")] + public async Task Q3b_StoreBackedLogRelaysTheSkippedPutAsAPut() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-" + Guid.NewGuid().ToString("N")); + var cache = new CultCache(); + var replica = Db(cache, primary: false, [NoteSchema(cache)], store: new CultNetFileShardMutationLogStore(dir)); + await replica.ApplyShardLogResponseAsync(UnownedPutLog(replica, "relay:k")); + var relayed = replica.GetMutationLogMessages(ShardId); + var memory = replica.GetMutationLog(ShardId); + TestContext.Out.WriteLine($"store relay: kind={relayed[0].ChangeKind} put={relayed[0].Put != null}; memory entry kind={memory[0].Kind} doc={(memory[0].Document == null ? "null" : "set")}"); + Assert.That(relayed[0].ChangeKind, Is.EqualTo("added")); + } + + // Q3c: a chained replica that does own the row (config drift) and holds it deletes it on the relayed entry. + [Test, Category("SoulFinding")] + public async Task Q3c_AChainedReplicaThatOwnsTheRowDeletesWhatThePrimaryPut() + { + var cacheA = new CultCache(); + var a = Db(cacheA, primary: false, [NoteSchema(cacheA)]); + await a.ApplyShardLogResponseAsync(UnownedPutLog(a, "relay:k")); + + var cacheC = new CultCache(); + var c = Db(cacheC, primary: false, [NoteSchema(cacheC), MeshSchema(cacheC)]); + await cacheC.UpsertAsync(new MeshQuickstartNote { NoteId = "p", Body = "earlier" }, new CultRecordHandle(new CultRecordKey("relay:k"))); + await c.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + MessageId = "chain", + ShardId = ShardId, + ShardEpoch = 1, + Entries = a.GetMutationLogMessages(ShardId).ToArray() + }); + var row = cacheC.Get(new CultRecordKey("relay:k")); + TestContext.Out.WriteLine($"chained row after relay: {row?.Body ?? ""}"); + Assert.That(row, Is.Not.Null, "the primary put this row; a relay must not delete it"); + } + + // A skipped entry is relayed exactly as the primary sent it, whether the replica keeps its log in memory or on disk. + [Test] + public async Task SkippedEntriesAreRelayedIdenticallyFromMemoryAndFromDisk() + { + var dir = Path.Combine(Path.GetTempPath(), "soulp4d-parity-" + Guid.NewGuid().ToString("N")); + try + { + var memoryCache = new CultCache(); + var inMemory = Db(memoryCache, primary: false, [NoteSchema(memoryCache)]); + var diskCache = new CultCache(); + var onDisk = Db(diskCache, primary: false, [NoteSchema(diskCache)], store: new CultNetFileShardMutationLogStore(dir)); + var log = UnownedPutLog(inMemory, "relay:k"); + var removal = new CultNetShardLogEntryMessage + { + Sequence = 2, + CommittedAt = Now, + ChangeKind = "removed", + Delete = new CultNetDocumentDeleteMessage + { + MessageId = "d", + SchemaId = memoryCache.Registry.GetRequired().SchemaId, + RecordKey = "relay:k", + ShardId = ShardId, + ShardEpoch = 1 + } + }; + log.Entries = [log.Entries[0], removal]; + + await inMemory.ApplyShardLogResponseAsync(log); + await onDisk.ApplyShardLogResponseAsync(log); + + var memory = inMemory.GetMutationLogMessages(ShardId); + var disk = onDisk.GetMutationLogMessages(ShardId); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(new[] { "added", "removed" })); + Assert.That(memory.Select(e => e.ChangeKind), Is.EqualTo(disk.Select(e => e.ChangeKind))); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(new[] { true, false })); + Assert.That(memory.Select(e => e.Put != null), Is.EqualTo(disk.Select(e => e.Put != null))); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(new[] { false, true })); + Assert.That(memory.Select(e => e.Delete != null), Is.EqualTo(disk.Select(e => e.Delete != null))); + Assert.That(memory[0].Put!.Document!.RecordKey, Is.EqualTo(disk[0].Put!.Document!.RecordKey)); + Assert.That(memory[0].Put!.Document!.Payload, Is.EqualTo(disk[0].Put!.Document!.Payload), "the document bytes are the primary's"); + } + finally + { + if (Directory.Exists(dir)) Directory.Delete(dir, true); + } + } + + // Q2: key-prefixed shard: the schema is owned, a key outside the prefix is not. + [Test, Category("SoulPin")] + public async Task Q2_KeyPrefixedReadsServeOnlyTheOwnedKeys() + { + var cache = new CultCache(); + var database = Db(cache, primary: true, [NoteSchema(cache)], keyPrefix: "a:"); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "a" }, new CultRecordHandle(new CultRecordKey("a:1"))); + await cache.UpsertAsync(new NetworkSchemaNote { Text = "b" }, new CultRecordHandle(new CultRecordKey("b:1"))); + Assert.That(database.GetAll().Select(n => n.Text), Is.EqualTo(new[] { "a" })); + Assert.That(await database.GetAsync(new CultRecordKey("b:1")), Is.Null); + Assert.That(await database.GetAsync(new CultRecordKey("a:1")), Is.Not.Null); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var snapshot = databaseServer.CreateSnapshotResponse(new CultNetSnapshotRequestMessage { MessageId = "all" }); + Assert.That(snapshot.Documents.Select(d => d.RecordKey), Is.EqualTo(new[] { "a:1" })); + } + + [Test, Category("SoulPin")] + public async Task Q2_PrimaryAllServesEveryCachedRow() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions { Shards = [CultNetShardDescriptor.PrimaryAll()] }); + await cache.UpsertAsync(new MeshQuickstartNote { NoteId = "x", Body = "x" }, new CultRecordHandle(new CultRecordKey("any:1"))); + Assert.That(database.GetAll().Count(), Is.EqualTo(1)); + Assert.That(database.GetByName("x"), Is.Not.Null); + } + + // Q2 cost: GetAll through the database versus the cache, scoped shard (reflection per row). + [Test, Category("SoulMeasure"), Explicit("Measurement, no assertion.")] + public void Q2_GetAllCost() + { + var cache = new CultCache(); + var database = Db(cache, primary: false, [NoteSchema(cache)], keyPrefix: "n:"); + cache.Commit(batch => + { + for (var i = 0; i < 20000; i++) + batch.Upsert(new NetworkSchemaNote { Text = "t" + i }, new CultRecordHandle(new CultRecordKey("n:" + i))); + }); + _ = cache.GetAll().Count(); + _ = database.GetAll().Count(); + var sw = Stopwatch.StartNew(); + for (var r = 0; r < 5; r++) _ = cache.GetAll().Count(); + var cacheMs = sw.Elapsed.TotalMilliseconds / 5; + sw.Restart(); + for (var r = 0; r < 5; r++) _ = database.GetAll().Count(); + var dbMs = sw.Elapsed.TotalMilliseconds / 5; + TestContext.Out.WriteLine($"GetAll 20000 rows: cache={cacheMs:F2}ms database={dbMs:F2}ms ratio={dbMs / Math.Max(cacheMs, 0.001):F1}x"); + } + + // Wire: the bytes a peer receives for the unowned refusal. + [Test, Category("SoulMeasure"), Explicit("Prints the wire bytes the Rust captured-bytes case pins.")] + public void Wire_UnownedSchemaBytes() + { + var message = CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("some.schema.v1", new CultRecordKey("some:key"))); + var bytes = MessagePack.MessagePackSerializer.Serialize(message, CultNetSchemaMessageSerialization.Options); + TestContext.Out.WriteLine("HEX=" + Convert.ToHexString(bytes)); + TestContext.Out.WriteLine("JSON=" + MessagePack.MessagePackSerializer.ConvertToJson(bytes)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/SoulPub3Tests.cs b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs new file mode 100644 index 00000000..dacfd470 --- /dev/null +++ b/tests/GameCult.Networking.Tests/SoulPub3Tests.cs @@ -0,0 +1,276 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using R3; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + [CultDocument("tests.soul_throwing_name", "tests.soul_throwing_name.v1")] + [MessagePack.MessagePackObject] + public sealed class SoulThrowingName + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + + [MessagePack.Key(1)] + [CultName] + public string Name + { + get => Text == "boom" ? throw new InvalidOperationException("name getter boom") : Text; + set { } + } + } + + [NonParallelizable] + public sealed class SoulPub3Tests + { + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5); + private static NetworkSchemaNote Note(string text) => new() { Schema = "tests.networking_note.v1", Text = text }; + private static string SchemaId => CultDocumentRegistry.Shared.GetRequired().SchemaId; + + // Another thread becomes the drainer and parks inside a subscriber on the change "slow". + private static (Task Writer, ManualResetEventSlim Release) BlockDrainer(CultNetDatabase database, CultCache cache, out IDisposable subscription) + { + var entered = new ManualResetEventSlim(); + var release = new ManualResetEventSlim(); + subscription = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "slow") + { + entered.Set(); + release.Wait(Wait); + } + }); + var writer = Task.Run(() => cache.UpsertAsync(Note("slow"), new CultRecordHandle(new CultRecordKey("soul3:slow"))).GetAwaiter().GetResult()); + Assert.That(entered.Wait(Wait), Is.True, "drainer parked"); + return (writer, release); + } + + // P1: a write that returns has been logged and published. + [Test] + public async Task P1_AWriteThatReturnedIsLoggedAndPublished() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue<(string? Text, int Thread)>(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue((change.Document?.Text, Environment.CurrentManagedThreadId))); + var (writer, release) = BlockDrainer(database, cache, out var block); + var fast = new CultRecordKey("soul3:fast"); + var writerThread = 0; + await Task.Run(async () => { writerThread = Environment.CurrentManagedThreadId; await database.PutAsync(fast, Note("fast")); }); + + var logged = database.LastWriteSequence(SchemaId, fast); + var published = seen.Any(entry => entry.Text == "fast"); + release.Set(); + await writer; + block.Dispose(); + var deliveredOn = seen.First(entry => entry.Text == "fast").Thread; + TestContext.Progress.WriteLine($"SOUL P1 logged-at-return={logged} published-at-return={published} writer={writerThread} deliveredOn={deliveredOn}"); + Assert.Multiple(() => + { + Assert.That(logged, Is.Not.Null, "PutAsync returned before its change was logged"); + Assert.That(published, Is.True, "PutAsync returned before its change was published"); + }); + } + + // P2: a subscriber that waits (bounded) for another thread's write to be observed. + [Test] + public async Task P2_ASubscriberWaitingForAnotherThreadsWriteIsNotStarved() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var other = new ManualResetEventSlim(); + var observedInTime = false; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text == "other") + other.Set(); + if (change.Document?.Text == "trigger") + { + Task.Run(() => database.PutAsync(new CultRecordKey("soul3:other"), Note("other"))); + observedInTime = other.Wait(TimeSpan.FromSeconds(2)); + } + }); + await database.PutAsync(new CultRecordKey("soul3:trigger"), Note("trigger")); + Assert.That(observedInTime, Is.True, "another thread's committed write never reached subscribers while this handler ran"); + } + + // P3: one observer's escaped exception (fail-fast R3 handler) loses one change; it must not stall all later ones. + [Test] + public async Task P3_AnEscapedObserverExceptionDoesNotStallLaterChanges() + { + var previous = ObservableSystem.GetUnhandledExceptionHandler(); + ObservableSystem.RegisterUnhandledExceptionHandler(exception => throw exception); + try + { + var cache = new CultCache(); + using var thrower = cache.Watch().Subscribe(change => + { + if (change.Document?.Text == "boom") throw new InvalidOperationException("observer boom"); + }); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try { await cache.UpsertAsync(Note("boom"), new CultRecordHandle(new CultRecordKey("soul3:boom"))); } + catch (InvalidOperationException) { } + + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + var after = new CultRecordKey("soul3:after"); + await database.PutAsync(after, Note("after")); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + finally + { + ObservableSystem.RegisterUnhandledExceptionHandler(previous); + } + } + + // P4: an admission that throws inside Apply after minting a Sequence (a [CultName] getter) must not stall later changes. + [Test] + public async Task P4_AThrowMidApplyDoesNotStallLaterChanges() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var seen = new ConcurrentQueue(); + using var watch = database.Watch().Subscribe(change => seen.Enqueue(change.Document?.Text)); + try + { + cache.Commit(batch => + { + batch.Upsert(new SoulThrowingName { Text = "ok" }, new CultRecordHandle(new CultRecordKey("soul3:ok"))); + batch.Upsert(new SoulThrowingName { Text = "boom" }, new CultRecordHandle(new CultRecordKey("soul3:boomname"))); + }); + } + catch (InvalidOperationException) { } + + var after = new CultRecordKey("soul3:after-apply"); + await database.PutAsync(after, Note("after")); + TestContext.Progress.WriteLine($"SOUL P4 ok-in-cache={cache.Get(new CultRecordKey("soul3:ok")) != null}"); + Assert.Multiple(() => + { + Assert.That(seen, Does.Contain("after"), "a later commit was never published"); + Assert.That(database.LastWriteSequence(SchemaId, after), Is.Not.Null, "a later commit was never logged"); + }); + } + + // P5: writers no longer carry their own delivery; the backlog behind one drainer grows without bound. + [Test] + public async Task P5_OtherWritersDoNotPileAnUnboundedBacklogOnOneDrainer() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache); + var delivered = 0; + var backlogAtBDone = -1; + const int bWrites = 2000; + using var watch = database.Watch().Subscribe(change => + { + if (change.Document?.Text?.StartsWith("b") == true) Interlocked.Increment(ref delivered); + if (change.Document?.Text == "a") + { + Task.Run(async () => + { + for (var i = 0; i < bWrites; i++) + await database.PutAsync(new CultRecordKey($"soul3:b:{i}"), Note($"b{i}")); + }).Wait(Wait); + backlogAtBDone = bWrites - Volatile.Read(ref delivered); + } + }); + await database.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + TestContext.Progress.WriteLine($"SOUL P5 undelivered after B's {bWrites} writes returned: {backlogAtBDone}"); + Assert.That(backlogAtBDone, Is.EqualTo(0)); + } + + // P6: a prediction still queued when the authoritative put arrives. + [Test] + public async Task P6_AnAuthoritativePutArrivingBeforeItsPredictionIsDeliveredStillReconciles() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + RuntimeId = "local", + ClientAuthorityScopes = [new CultNetClientAuthorityScope("local")] + }); + var key = new CultRecordKey("soul3:predict"); + var kinds = new ConcurrentQueue(); + using var watch = database.WatchRecord(key).Subscribe(change => kinds.Enqueue(change.Kind)); + CultNetDocumentPutRawMessage Put(string text) + { + var message = database.Documents.CreateRawDocumentPutMessage(text, new CultRecordHandle(key), Note(text)); + message.ShardId = database.Shards[0].ShardId; + message.ShardEpoch = database.Shards[0].Epoch; + return message; + } + + var (writer, release) = BlockDrainer(database, cache, out var block); + await database.PutPredictedAsync(key, Note("predicted")); + await database.ApplyPutAsync(Put("auth1")); + release.Set(); + await writer; + block.Dispose(); + await database.ApplyPutAsync(Put("auth2")); + + Assert.That(kinds, Is.EqualTo(new[] + { + CultNetDatabaseChangeKind.Predicted, + CultNetDatabaseChangeKind.Reconciled, + CultNetDatabaseChangeKind.Updated + })); + } + + // P7: a replica's absent-key delete is logged ahead of an earlier entry still queued; a bounded pull then sees a gap. + [Test] + public async Task P7_AReplicaLogStaysInSequenceOrderWhenAnAbsentDeleteOvertakesAQueuedPut() + { + const string shardId = "chain3"; + CultNetDatabase Make(CultCache cache, bool primary) => new(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(shardId, "p", epoch: 2, isPrimary: primary, schemaIds: [SchemaId])] + }); + var p = Make(new CultCache(), true); + var rCache = new CultCache(); + var r = Make(rCache, false); + var c = Make(new CultCache(), false); + async Task Pull(CultNetDatabase from, CultNetDatabase to, int? limit = null) + { + await to.ApplyShardLogResponseAsync(new CultNetShardLogResponseMessage + { + ShardId = shardId, + ShardEpoch = 2, + Entries = from.GetMutationLogMessages(shardId, to.GetAppliedShardSequence(shardId), limit).ToArray() + }); + } + + var z = new CultRecordKey("soul3:z"); + await p.PutAsync(z, Note("z")); + await Pull(p, r); + await Pull(r, c); + rCache.Remove(z); + await p.PutAsync(new CultRecordKey("soul3:a"), Note("a")); + await p.DeleteAsync(z); + + var (writer, release) = BlockDrainer(r, rCache, out var block); + await Pull(p, r); + release.Set(); + await writer; + block.Dispose(); + + TestContext.Progress.WriteLine($"SOUL P7 r log order: {string.Join(",", r.GetMutationLog(shardId).Select(e => e.Sequence))}"); + Assert.Multiple(() => + { + Assert.That(r.GetMutationLog(shardId).Select(entry => entry.Sequence), Is.EqualTo(new[] { 1L, 2L, 3L })); + Assert.DoesNotThrowAsync(() => Pull(r, c, limit: 1), "a bounded pull from R"); + }); + } + } +} diff --git a/tests/vectors/document-variants-c2a/readability/manifest.txt b/tests/vectors/document-variants-c2a/readability/manifest.txt index 7d9b4224..cc89d63b 100644 --- a/tests/vectors/document-variants-c2a/readability/manifest.txt +++ b/tests/vectors/document-variants-c2a/readability/manifest.txt @@ -1,7 +1,7 @@ # Whether each runtime's own reader opens the file. A flush and a commit replace a file exactly when it reads. # Columns: file, csharp, rust, typescript, python. `../v3-base.msgpack` is the one valid store. ../v3-base.msgpack reads reads reads reads -zero-byte.bin reads reads reads reads +zero-byte.bin refuses refuses refuses refuses empty-array.bin reads reads reads reads legacy-envelopes.bin refuses reads reads reads legacy-trailing.bin refuses refuses refuses refuses